def extractData(self): self.training_data, self.training_outputs = parsing.parse(self.training_path, self.input_start_column, self.input_end_column, qualitative = self.qualitative_outputs, output_column = self.output_column) self.testing_data, self.testing_outputs = parsing.parse(self.testing_path, self.input_start_column, self.input_end_column, qualitative = self.qualitative_outputs, output_column = self.output_column) self.input_start_column = 0 self.input_end_column = len(self.training_data[0]) - 1 return 1 if self.qualitative_outputs: if self.testing_path == None: full_training = np.genfromtxt(self.training_path, delimiter=self.delimitation) full_size = len(full_training) random.shuffle(full_training) self.training_data = full_training[:full_size-self.number_of_tests] self.testing_data = full_training[full_size-self.number_of_tests:] else: self.training_data = np.genfromtxt(self.training_path, delimiter=self.delimitation) self.testing_data = np.genfromtxt(self.testing_path, delimiter=self.delimitation) max_output = max(self.getMaxOutput(self.training_data), self.getMaxOutput(self.testing_data)) self.training_outputs, self.training_outputs = self.getQualitativeOutputs(self.training_data, max_output) self.testing_outputs, self.testing_outputs = self.getQualitativeOutputs(self.testing_data, max_output) self.training_data = self.training_data[:,[x for x in xrange(self.input_start_column, self.input_end_column+1)]] self.testing_data = self.testing_data[:,[x for x in xrange(self.input_start_column,self.input_end_column+1)]] else: self.training_data = np.genfromtxt(self.training_path, delimiter=self.delimitation) self.testing_data = np.genfromtxt(self.testing_path, delimiter=self.delimitation) self.training_outputs = self.training_data[:,[self.output_column]] self.testing_outputs = self.testing_data[:,[self.output_column]] self.training_data = self.training_data[:,[x for x in xrange(self.input_start_column, self.input_end_column+1)]] self.testing_data = self.testing_data[:,[x for x in xrange(self.input_start_column,self.input_end_column+1)]]
def file_chooser(): title = "seleeect" filetypes = (('Audit files', '*.audit'), ('all files', '*.*')) global fname fname = filedialog.askopenfilename(initialdir='.', title=title, filetypes=filetypes) file_name = fname.split('/')[-1] parse(fname) update_label(file_name)
def main(argv=None): try: if argv is None: argv = sys.argv[1:] parser = _mk_options_parser() options, args = parser.parse_args(argv) if len(args) == 0: parser.error("insufficient arguments, expected at least one path.") return 2 res = parsing.parse(args, options.prune, options.mintime) if options.interactive or options.output == None: gui.show(res) else: filename = _get_filename(options.output) res_list = parsing.split_res(res, options.num) n = 1 for r in res_list: if len(res_list) == 1: f = filename + "." + options.format else: f = filename + "_" + str(n) + "." + options.format n = n + 1 batch.render(r, options.format, f) print "bootchart written to", f return 0 except parsing.ParseError, ex: print("Parse error: %s" % ex) return 2
def get_tests(self, the_makefile, the_dict): with open(the_makefile) as f: input = f.read() output = parse(input) apps = [each for each in output if 'Command' in str(type(each))] otb_tests = [each for each in apps if 'ADD_TEST' in each.name.upper()] return otb_tests
def parseData(ticker): modelsFile = open('models.pkl', 'rb') m0, m1 = pickle.load(modelsFile) F, data = dataXY(ticker) ptable = likTable(F, m0, m1) res = parse(ptable) kps = data[data.pivots!=0].Close idx = kps.index fig = plt.figure(figsize=(15,10)) ax = fig.add_subplot(211) bx = fig.add_subplot(212) kps.plot(style='--o', ax=ax) ptable.plot(style='o', ax=bx) #import ipdb; ipdb.set_trace() for terminal, (xmin, xmax), _, logLik in res: center = idx[(xmin+xmax)/2] high = data[idx[xmin]:idx[xmax]].Close.max() low = data[idx[xmin]:idx[xmax]].Close.min() bottom, top = ax.get_ylim() llow = (low-bottom)/float(top-bottom) hhigh = (high-bottom)/float(top-bottom) ax.axvspan(idx[xmin], idx[xmax], ymin=llow, ymax=hhigh, alpha=0.3) ax.annotate("%s"%(terminal), (center, high)) #ax.annotate("%s[%.3f]"%(terminal,logLik), (center, high)) bx.legend(loc=4) plt.savefig("parsed/%s.png"%(ticker))
def main(): # Parse arguments parser = argparse.ArgumentParser( description='Pretty-print CMakeLists files.') parser.add_argument('files', type=str, nargs='*', help='files to pretty print (default is stdin)') parser.add_argument('-t', '--tree', action='store_true', help='print out the syntax trees') args = parser.parse_args() # Gather files filenames = args.files files = [('<stdin>', sys.stdin)] if filenames: files = [(name, open(name)) for name in filenames] # Process files for (name, file) in files: with file: input = file.read() tree = cmp.parse(input, path=name) if args.tree: # Print out AST print(repr(tree)) else: # Pretty print print(str(tree), end='')
def main(): parser = argparse.ArgumentParser(description="Halite II training") parser.add_argument("--model_name", help="Name of the model") parser.add_argument("--minibatch_size", type=int, help="Size of the minibatch", default=100) parser.add_argument("--steps", type=int, help="Number of steps in the training", default=100) parser.add_argument("--data", help="Data directory or zip file containing uncompressed games") parser.add_argument("--cache", help="Location of the model we should continue to train") parser.add_argument("--games_limit", type=int, help="Train on up to games_limit games", default=1000) parser.add_argument("--seed", type=int, help="Random seed to make the training deterministic") parser.add_argument("--bot_to_imitate", help="Name of the bot whose strategy we want to learn") parser.add_argument("--dump_features_location", help="Location of hdf file where the features should be stored") args = parser.parse_args() # Make deterministic if needed if args.seed is not None: np.random.seed(args.seed) nn = NeuralNet(cached_model=args.cache, seed=args.seed) if args.data.endswith('.zip'): raw_data = fetch_data_zip(args.data, args.games_limit) else: raw_data = fetch_data_dir(args.data, args.games_limit) data_input, data_output = parse(raw_data, args.bot_to_imitate, args.dump_features_location) data_size = len(data_input) training_input, training_output = data_input[:int(0.85 * data_size)], data_output[:int(0.85 * data_size)] validation_input, validation_output = data_input[int(0.85 * data_size):], data_output[int(0.85 * data_size):] training_data_size = len(training_input) # randomly permute the data permutation = np.random.permutation(training_data_size) training_input, training_output = training_input[permutation], training_output[permutation] print("Initial, cross validation loss: {}".format(nn.compute_loss(validation_input, validation_output))) curves = [] for s in range(args.steps): start = (s * args.minibatch_size) % training_data_size end = start + args.minibatch_size training_loss = nn.fit(training_input[start:end], training_output[start:end]) if s % 25 == 0 or s == args.steps - 1: validation_loss = nn.compute_loss(validation_input, validation_output) print("Step: {}, cross validation loss: {}, training_loss: {}".format(s, validation_loss, training_loss)) curves.append((s, training_loss, validation_loss)) cf = pd.DataFrame(curves, columns=['step', 'training_loss', 'cv_loss']) fig = cf.plot(x='step', y=['training_loss', 'cv_loss']).get_figure() # Save the trained model, so it can be used by the bot current_directory = os.path.dirname(os.path.abspath(__file__)) model_path = os.path.join(current_directory, os.path.pardir, "models", args.model_name + ".ckpt") print("Training finished, serializing model to {}".format(model_path)) nn.save(model_path) print("Model serialized") curve_path = os.path.join(current_directory, os.path.pardir, "models", args.model_name + "_training_plot.png") fig.savefig(curve_path)
def compute_predictions(rules, non_terminals, unary_rules_proba, binary_rules_proba, transition_rules_list, vocabulary, embeddings, test_corpus, filename='evaluation_data.parser_output'): """ Creates prediction file from test corpus (for pyevalb) """ n_test = len(test_corpus) with open(filename, 'w') as f: for i, elem in enumerate(test_corpus): print('{} elements have been processed out of {}'.format( i + 1, n_test)) parsed = parsing.parse(rules, non_terminals, unary_rules_proba, binary_rules_proba, transition_rules_list, vocabulary, embeddings, tree=elem) f.write("%s\n" % parsed)
def ep(input, debug=True): ast = parse(input) if debug: print ast val = evaluate(ast) if val is not None and debug: print(_to_scheme_str(val))
def get_apps(self, the_makefile, the_dict): with open(the_makefile) as f: input = f.read() output = parse(input) apps = [each for each in output if "Command" in str(type(each))] otb_apps = [each for each in apps if "OTB_TEST_APPLICATION" in each.name.upper()] return otb_apps
def analyze_macro_application(astnode): exp = astnode.exp macro, macro_args = exp[0].exp, astnode.get_exp() expanded = macro_expand(macro, macro_args) tostring = str(to_string(expanded)) parsed = analyze(list(parse(tostring))[-1]) return CodeObject(astnode, lambda env: parsed.exec_(env))
def fetch(): display() parsing.parse() parsing.new_parse() T.config(state="normal") try: mac_file = open('mac_address.txt', 'r') T.delete(1.0, END) T.insert(INSERT, "MAC file found\n\n") mac_addr = mac_file.readlines() for mac in mac_addr: fileMenu.add_command(label=mac.rstrip('\n'), command=partial(sta_data, mac.rstrip('\n'))) except IOError: T.insert(INSERT, "MAC file not found\n\n") T.config(state="disabled")
def valid_proof(expressions): """ Takes in a list of parse tree expressions [A, B, C, ... Z]. Outputs one parse tree ((A&B&C&...) -> Z). Last expression is the conclusion, all others are premises. """ trees = [parse(e) for e in expressions] argument = serialize_argument_trees(trees) return all([row.value for row in truthtable.from_tree(argument)])
def get_apps(self, the_makefile, the_dict): input = open(the_makefile).read() output = parse(input) apps = [each for each in output if 'Command' in unicode(type(each))] otb_apps = [ each for each in apps if 'OTB_TEST_APPLICATION' in each.name.upper() ] return otb_apps
def main(argv=None): try: if argv is None: argv = sys.argv[1:] parser = _mk_options_parser() options, args = parser.parse_args(argv) writer = _mk_writer(options) if len(args) == 0: print "No path given, trying /var/log/bootchart.tgz" args = [ "/var/log/bootchart.tgz" ] res = parsing.parse(writer, args, options.prune, options.crop_after, options.annotate) if options.interactive or options.output == None: gui.show(res, options) elif options.boottime: import math proc_tree = res[3] if proc_tree.idle: duration = proc_tree.idle else: duration = proc_tree.duration dur = duration / 100.0 print '%02d:%05.2f' % (math.floor(dur/60), dur - 60 * math.floor(dur/60)) else: if options.annotate_file: f = open (options.annotate_file, "w") try: for time in res[4]: if time is not None: # output as ms print >> f, time * 10 else: print >> f finally: f.close() filename = _get_filename(args, options) def render(): batch.render(writer, res, options, filename) if options.profile: import cProfile import pstats profile = '%s.prof' % os.path.splitext(filename)[0] cProfile.runctx('render()', globals(), locals(), profile) p = pstats.Stats(profile) p.strip_dirs().sort_stats('time').print_stats(20) else: render() return 0 except parsing.ParseError, ex: print("Parse error: %s" % ex) return 2
def assertParseContains(self, location, contains): # Because the parser is overly greedy and gives many possible # responses, it keeps the unit tests tidier and less brittle if we # just list one important parse result that we expect, rather than # listing every single test result. try: actual = [dict(result) for result in parse(location)] except ParsingError, e: self.fail(e)
def intro(): answer = askForever(phrasebase.getReply("greetings")) meaning = parsing.parse(answer) if "greet" in meaning: ui.tell("Nice to meet you!") answer = askForever("You are a human, aren't you?") meaning = parsing.parse(answer) if meaning == "yes": ui.tell("Great! This means you can help me.") if meaning == "no": answer = askForever("Are you sure? What is the sum of two and three?") meaning = parsing.parse(answer) if meaning == "five": ui.tell("See, you solved the captcha. You are sufficiently human for my purposes.") else: ui.tell("Very funny. You are definitely human.") ui.tell("My programmers want me to teach you how to be rational, make good decisions and judge situations correctly.") askForever("Do you want to be more rational?") meaning = parsing.parse(answer) if meaning == "yes": ui.tell("Yeah, that's the spirit!") if meaning == "no": answer = askForever("Why would you think that? Rationality is just the ability to make good decisions. Do you want to be able to make good decisions?") meaning = parsing.parse(answer) if meaning == "no": askbreak() ui.tell("I will just try to ask you some questions, and try to explain to you what you could do better. If I do a bad job at explaning, just ask me, ok? I never taught humans before.") ui.tell("So, let's see... the first thing I want you to know is that you don't have to be extremely intelligent to be rational. There are very intelligent people who do things that are not at all reasonable. The key to rational decisions is to know when not to follow your gut feelings, but to stop and actually think about the problem."); answer = askForever("To get used to the whole situation - how about I ask you a test question? Just to make sure I am doing this teaching thing right. ") meaning = parsing.parse(answer) if meaning == "yes": ui.tell("Okay, thank you!") if meaning == "no": ui.tell("I would nevertheless like to ask the test question.") answer = askForever("This is my first question: Do people need to follow their gut feelings to make rational decisions?") meaning = parsing.parse(answer) if meaning == "no": ui.tell("Amazing! I mean, it was easy, I know, but you did it. Very reasonable of you to say this! Now we can start with the actual teaching.") if meaning == "yes": ui.tell("Uhm... no. This is a bit awkward. Following you gut feelings means not to think about something, but just go with what feels right. A lot of psychologists have shown that people tend to make a lot of mistakes when they make decisions that way.") answer = askForever("Do you still want to continue?") meaning = parsing.parse(answer) if meaning == "yes": ui.tell("Okay! Let's start with the actual teaching!") else : askbreak()
def generate_operators(): repo_names = getRepoNames() new_operators = [] output = open('mutation_operators.txt','w') init.init() for repo in repo_names: print("collecting operators form repo : " + repo + " .....") diff_file = open('../resources/diffs_modified/'+repo+'_diff_modified','r') minus = None plus = None for line in diff_file: if minus is None: minus = line.strip() elif plus is None: plus = line.strip() if plus is not None: #parse both minus and plus if not (("'''" in minus) or ("'''" in plus) or ('"""' in minus) or ('"""' in plus)): token_minus = parse(minus[1:]) token_plus = parse(plus[1:]) new_operator = compareTok(token_minus, token_plus) if (new_operator is not None): if (new_operator in new_operators): init.duplicate += 1 else: output.write(str(new_operator)+ "\n") new_operators.append(new_operator) minus = None plus = None print("size : " + str(len(new_operators)) ); print("collecting done") #return new_operators print("less and equal Than 2 : " + str(init.lessThan2)) print("more and equal than 10 tokens : " + str(init.moreThan10)) print("same afterToken, beforeToken : " + str(init.same)) print("more and equal than 4 identifiers : " + str(init.moreThan4Iden)) print("3 identifers in a row : " + str(init.inArow)) print("unmatched bracket : " + str(init.unmatchedBracket)) print("duplicate : " + str(init.duplicate)) output.close()
def main(): update_id = last_update(get_updates_json(url))['update_id'] while True: if update_id == last_update(get_updates_json(url))['update_id']: update_id += 1 if last_update(get_updates_json(url))['message']['text'] == 'bet': result = parse() send_mess(get_chat_id(last_update(get_updates_json(url))), result) else: send_mess(get_chat_id(last_update(get_updates_json(url))), 'test') result = parse() if (result != "NOW NOTHING"): send_mess(get_chat_id(last_update(get_updates_json(url))), result) time.sleep(3)
def startliza: parsing.setui(ui) warnings.filterwarnings(module='sklearn*', action='ignore', category=DeprecationWarning) while(True): ui.tell("Say something. I will try to understand and parse it.") answer = ui.listen() parsed = parsing.parse(answer) analysis = str(parsed['intent']['name']) + ", with confidence " + str(parsed['intent']['confidence']) ui.analyze(analysis)
def eval_string(inport): "Eval string or inport" x = parse(inport) if x is eof_object: return x val = lisp_eval(x) if val is None: return None return to_string(val)
def __init__(self, **kwargs): total_start = datetime.now() test = kwargs['filename'] or input('\nEnter test name: ') test_rating = parse(os.path.join(TEST_DIR, test + '.test')) base_rating = parse(os.path.join(TEST_DIR, test + '.base')) graph = Graph(base_rating=base_rating) while True: try: cluster_number = kwargs['cluster_num'] or int(input('\nEnter number of clusters: ')) component = BoruvkaMST(graph, cluster_number) break except (ClusterNumberException, ValueError): print('Number of clusters must be > 0') continue predictor = Predictor(component, base_rating) from copy import deepcopy predicted = deepcopy(test_rating) unpredictables = 0 start = datetime.now() with open(os.path.join(TEST_DIR, test + '.res'), newline='', mode='w') as csv_file: data_writer = csv.writer(csv_file, delimiter=' ', quotechar='|') for user in test_rating.keys(): for movie in test_rating[user].keys(): predicted[user][movie] = None predicted[user][movie] = predictor.predict(user, movie) if not predicted[user][movie]: unpredictables += 1 data_writer.writerow([user, movie, predicted[user][movie]]) self.predicting_time = datetime.now() - start print('Time for predicting:', self.predicting_time, end='\n\n') self.total_time = datetime.now() - total_start print('Total time:', self.total_time) print('Number of unpredictable users:', unpredictables) self.evaluator = Evaluator(test_rating, predicted)
def eval(_str, env=THE_GLOBAL_ENV): """ Loispy eval (overwrites python eval) Parse a string, analyze all the expressions in the context of a given environment, and return the last resulting value @param str _str @param Environment env """ return [analyze(e, toplevel=True).exec_(env) for e in parse(_str)][-1]
def test_serialize_argtree_simple(): expressions = ["A", "(A>B)", "B"] trees = [parse(e) for e in expressions] argument = serialize_argument_trees(trees) assert_is_instance(argument, IfNode) assert_is_instance(argument.l, AndNode) assert_is_instance(argument.r, AtomNode) assert_is_instance(argument.l.l, AtomNode) assert_is_instance(argument.l.r, IfNode) assert_is_instance(argument.l.r.l, AtomNode) assert_is_instance(argument.l.r.r, AtomNode)
def print_truth_table(exp, verbose, output=True): """Outputs the truth table for an expression to stdout.""" try: tree = parsing.parse(exp) table = truth_table(exp) except IOError as e: print("Parse error: %s" % e) return for row in table: print(row) if verbose: print() table = truth_table(exp) print_sat_info(table)
def __init__(self, report, selector): self.report_number = report self.original_report_number = report self.year = selector.report_list['year'][1] self.parsed = parse() #for parsing inputs from rows self.connection = connect() #connects to the DB self.queries = queries() #query text for all tables self.agg = agg() #aggregation functions for all tables self.json_builder = self.JSON_constructor_return() self.parse_return_string = self.parse_return() self.aggregation = self.aggregation_return(self.year, report) self.id_compiler = id_compiler() self.report_types = {'A':'Aggregate', 'D':'Disclosure', 'N':'National'}
def __init__(self, report, selector): self.report_number = report self.original_report_number = report self.year = selector.report_list['year'][1] self.parsed = parse() #for parsing inputs from rows self.connection = connect() #connects to the DB self.queries = queries() #query text for all tables self.agg = agg() #aggregation functions for all tables self.json_builder_return = self.JSON_constructor_return() self.parse_return_string = self.parse_return() self.aggregation = self.aggregation_return(self.year, report) self.id_compiler = id_compiler() self.report_types = {'A':'Aggregate', 'D':'Disclosure', 'N':'National'}
def run(local, remote, merged, conflict_resolution): print("TSCN-Merge") print(f"local/MINE: {local}") print(f"remote/THEIRS: {remote}") mine: TscnFile = parse(local) theirs: TscnFile = parse(remote) result: TscnFile = TscnFile() result.gd_scene = mine.gd_scene resolver = interactive if conflict_resolution == "mine": resolver = always_mine if conflict_resolution == "theirs": resolver = always_theirs # Step 1: Merge external resources and consolidate IDs. merge_ext_resources(mine, theirs, result) # Step 2: Merge sub resources merge_sub_resources(mine, theirs, result, resolver) # Step 3: Merge the node trees merge_nodes(mine, theirs, result, resolver) # Step 4: Merge connections merge_connections(mine, theirs, result) if not merged: print(result.to_string()) else: absolute_path = os.path.abspath(merged) directory, file_name = os.path.split(absolute_path) os.makedirs(directory, exist_ok=True) out_file = open(file_name, "w") out_file.write(result.to_string()) out_file.close() print(f"Result written to {absolute_path}.")
def askbreak(): ui.tell("Do you want to continue with this?") answer = ui.listen() if (answer == ""): ui.tell("Do you want to continue? If you don't reply, I have to assume that you are gone...") answer = ui.listen() if (answer == ""): ui.tell("Well, okay... I guess we can't continue then.") return False meaning = parsing.parse(answer) if meaning == "yes": ui.tell("Okay, great! So let's return to the questions.") return True ui.tell("Oh. I am sorry.") return False
def receive(self, echo = False): """ Receive messages """ msg=None while msg == None: try: pay = self.__sock.recv(1024) self.__buffer+=pay if not pay: break except socket.error,e: break self.__buffer, msg = p.parse(self.__buffer)
def main(): new_offset = None today = now.day hour = now.hour while True: # greet_bot.get_updates(new_offset) last_update = greet_bot.get_last_update() # last_update_id = last_update['update_id'] # last_chat_text = last_update['message']['text'] last_chat_id = last_update['message']['chat']['id'] # last_chat_name = last_update['message']['chat']['first_name'] result = parse() greet_bot.send_message(last_chat_id, result)
def add_make(self, previous_context, new_file): with open(new_file) as f: input = f.read() output = parse(input) apps = [each for each in output if 'Command' in str(type(each))] setcommands = [each for each in apps if 'SET' in each.name.upper()] stringcommands = [ each for each in apps if 'STRING' in each.name.upper() ] environment = previous_context def mini_clean(item): if item.startswith('"') and item.endswith('"') and " " not in item: return item[1:-1] return item new_env = {} for command in setcommands: key = command.body[0].contents ct = " ".join([item.contents for item in command.body[1:]]) ct = mini_clean(ct) if "$" in ct: values = Template(ct) else: values = ct new_env[key] = values for stringcommand in stringcommands: key = stringcommand.body[-1].contents ct = stringcommand.body[-2].contents ct = mini_clean(ct.lower()) if "$" in ct: values = LowerTemplate(ct) else: values = ct new_env[key] = values resolve_dict(environment, new_env) environment.update(new_env) return environment
def __init__(self, expression): # Parse any raw string expressions. assert type(expression) is not type(self) if type(expression) is str: expression = parse(expression) # Use immutable data structures for speed & safety. if type(expression) is list: expression = tuple(expression) self._expression = expression self._length = None self._string = None self._length = self._calculate_length() self._string = self._calculate_string() self.height = self._calculate_height()
def test_s3_cloudtrail_pasing_and_enrichment(self, lambda_boto3, parsing_boto3): context = Context() boto3 = parsing_boto3.client() boto3.get_object.return_value = {"Body": test_data_gzipped()} payload = { "s3": { "bucket": { "name": "test-bucket", }, "object": { "key": "601427279990_CloudTrail_us-east-1_20210503T0000Z_QrttGEk4ZcBTLwj5.json.gz" }, } } result = parsing.parse({"Records": [payload]}, context) expected = copy.deepcopy([test_data["Records"][0]]) expected[0].update( { "ddsource": "cloudtrail", "ddsourcecategory": "aws", "service": "cloudtrail", "aws": { "s3": { "bucket": payload["s3"]["bucket"]["name"], "key": payload["s3"]["object"]["key"], }, "function_version": context.function_version, "invoked_function_arn": context.invoked_function_arn, }, } ) # yeah, there are tags, but we don't care to compare them result[0].pop("ddtags") # expected parsed result, now testing enrichment self.assertEqual(expected[0], result[0]) expected[0]["host"] = "i-08014e4f62ccf762d" self.assertEqual(expected[0], lambda_function.enrich(result)[0])
def get_apps_with_context(self, the_makefile, the_dict): input = open(the_makefile).read() output = parse(input) def is_a_command(item): return 'Command' in unicode(type(item)) appz = [] context = [] for each in output: if is_a_command(each): if 'FOREACH' in each.name and 'ENDFOREACH' not in each.name: args = [item.contents for item in each.body] context.append(args) elif 'ENDFOREACH' in each.name: context.pop() elif 'OTB_TEST_APPLICATION' in each.name.upper(): appz.append((each, context[:])) return appz
def add_make(self, previous_context, new_file): with open(new_file) as f: input = f.read() output = parse(input) apps = [each for each in output if 'Command' in str(type(each))] setcommands = [each for each in apps if 'SET' in each.name.upper()] stringcommands = [each for each in apps if 'STRING' in each.name.upper()] environment = previous_context def mini_clean(item): if item.startswith('"') and item.endswith('"') and " " not in item: return item[1:-1] return item new_env = {} for command in setcommands: key = command.body[0].contents ct = " ".join([item.contents for item in command.body[1:]]) ct = mini_clean(ct) if "$" in ct: values = Template(ct) else: values = ct new_env[key] = values for stringcommand in stringcommands: key = stringcommand.body[-1].contents ct = stringcommand.body[-2].contents ct = mini_clean(ct.lower()) if "$" in ct: values = LowerTemplate(ct) else: values = ct new_env[key] = values resolve_dict(environment, new_env) environment.update(new_env) return environment
def get_apps_with_context(self, the_makefile, the_dict): input = open(the_makefile).read() output = parse(input) def is_a_command(item): return 'Command' in str(type(item)) appz = [] context = [] for each in output: if is_a_command(each): if 'FOREACH' in each.name and 'ENDFOREACH' not in each.name: args = [item.contents for item in each.body] context.append(args) elif 'ENDFOREACH' in each.name: context.pop() elif 'OTB_TEST_APPLICATION' in each.name.upper(): appz.append((each, context[:])) return appz
def recv(self, echo=False): """ Tries to receive one complete message from the buffer. Returns None if client disconnected or if an error occurred. """ msg = None #continue to receive until at least one message is produced while msg == None: try: pay = self.__sock.recv(1024) self.__buffer += pay #client may have disconnected if not pay: break except socket.error, e: self.__logger.error("receive error: %s" % str(e)) break self.__buffer, msg = p.parse(self.__buffer)
def recv(self, echo = False): """ Tries to receive one complete message from the buffer. Returns None if client disconnected or if an error occurred. """ msg=None #continue to receive until at least one message is produced while msg==None: try: pay=self.__sock.recv(1024) self.__buffer+=pay #client may have disconnected if not pay: break except socket.error,e: self.__logger.error("receive error: %s"%str(e)) break self.__buffer,msg=p.parse(self.__buffer)
def start_composition(visual): cell = visual.head.cell while cell.label != '@': cell = cell.parent if cell is None: raise Exception("not implemented") context = cell.parent.rule.at(cell.parent.index(cell)) if isinstance(context, Context): rules = context.rules else: assert context rules = [context] # My attempt to create 'best result first' parser grand failed. results = list(parsing.parse([c.copy() for c in cell], rules, timeout=0.5)) results.sort(key=lambda x: x.badness) for result in results: new_block = result.wrap() replace(cell, new_block) visual.setpos(Position.bottom(new_block)) yield None visual.head, visual.tail = visual.document.undo()
def get_html(url): """Retrieve webpage content using provided URL. :param url: webpage address :type url: str :returns: Tuple with HTML content and size of the content :rtype: namedtuple(content, size) """ try: result = requests.get(url) logger.info('Fetching webpage: {url}'.format(url=url)) except requests.exceptions.RequestException: logger.exception('Fetching webpage failed: {url}'.format(url=url)) return None, None size = result.headers.get('content-length', len(result.content)) Result = namedtuple('Result', ['content', 'size']) return Result(content=parse(result.content), size='{:.1f}kb'.format(int(size) / 1024))
def main(): data = fetch(ticker='spy', start=datetime.datetime(2010, 1, 1), end=datetime.datetime(2012, 10, 1)) kps = data[zig(data)!=0].Close features = preProcess(kps) ptables = f2p(features) ptables.to_csv('test.csv') buildCSV(ptables) res = parse(ptables) idx = kps.index # fig = plt.figure(figsize=(10,10)) ax = fig.add_subplot(211) #data.Close.plot(style='.', ax=ax) kps.plot(style='--o', ax=ax) # bx = fig.add_subplot(212, sharex=ax) #features.plot(style='--o', ax=bx) ptables.plot(style='o', ax=bx) #import ipdb; ipdb.set_trace() for terminal, (xmin, xmax), _, _ in res: center = idx[(xmin+xmax)/2] high = data[idx[xmin]:idx[xmax]].Close.max() low = data[idx[xmin]:idx[xmax]].Close.min() bottom, top = ax.get_ylim() llow = (low-bottom)/float(top-bottom) hhigh = (high-bottom)/float(top-bottom) ax.axvspan(idx[xmin], idx[xmax], ymin=llow, ymax=hhigh, alpha=0.3) ax.annotate(terminal, (center, high)) if terminal in ( Terminal('hs', None, None), Terminal('ihs', None, None)) : ax.plot([idx[xmin+1], idx[xmax-1]], [kps[xmin+1], kps[xmax-1]], c='r' ) ax.plot([idx[xmin], idx[xmax]], [kps[xmin], kps[xmax]], c='r' ) ax.set_xlim([idx[0]-timedelta(weeks=1), idx[-1]]) plt.show()
def main(argv=None): try: if argv is None: argv = sys.argv[1:] parser = _mk_options_parser() options, args = parser.parse_args(argv) writer = _mk_writer(options) if len(args) == 0: print "No path given, trying /var/log/bootchart.tgz" args = [ "/var/log/bootchart.tgz" ] res = parsing.parse(writer, args, options.prune) if options.interactive or options.output == None: gui.show(res, options) else: filename = _get_filename(args, options) def render(): batch.render(writer, res, options, filename) if options.profile: import cProfile import pstats profile = '%s.prof' % os.path.splitext(filename)[0] cProfile.runctx('render()', globals(), locals(), profile) p = pstats.Stats(profile) p.strip_dirs().sort_stats('time').print_stats(20) else: render() return 0 except parsing.ParseError, ex: print("Parse error: %s" % ex) return 2
def test_CMakelists(self): provided = {} provided["OTB_SOURCE_DIR"] = self.root_dir provided["OTB_BINARY_DIR"] = self.build_dir provided["OTB_DATA_LARGEINPUT_ROOT"] = os.path.normpath(os.path.join(self.root_dir, "../OTB-Data/Input")) try: with open(os.path.join(self.root_dir, "CMakeLists.txt")) as file_input: content = file_input.read() output = parse(content) defined_paths = [each for each in output if 'Command' in str(type(each)) and "FIND_PATH" in each.name] the_paths = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in defined_paths} the_sets = [each for each in output if 'Command' in str(type(each)) and "SET" in each.name.upper()] the_sets = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in the_sets} the_sets = {key: " ".join(the_sets[key]) for key in the_sets} the_strings = set([each.body[-1].contents for each in output if 'Command' in str(type(each)) and "STRING" in each.name.upper()]) def mini_clean(item): if item.startswith('"') and item.endswith('"') and " " not in item: return item[1:-1] return item the_sets = {key: mini_clean(the_sets[key]) for key in the_sets} def templatize(item): if "$" in item: return Template(item) return item for key in the_sets: if key in the_strings: the_sets[key] = the_sets[key].lower() the_sets = {key: templatize(the_sets[key]) for key in the_sets} for path in the_paths: target_file = the_paths[path][1] suggested_paths = [] if len(the_paths[path]) > 2: suggested_paths = the_paths[path][2:] try: provided[path] = find_file(target_file) except Exception as e: for each in suggested_paths: st = Template(each) pac = os.path.abspath(st.safe_substitute(provided)) if os.path.exists(pac): provided[path] = pac break resolve_dict(provided, the_sets) provided.update(the_sets) return provided except Exception as e: traceback.print_exc() self.fail(str(e))
def testParseLogDir(self): res = parsing.parse(writer, [self.rootdir], False, None, None) self.assertEqual(6, len(res))
def parse(string): structure = parsing.parse(string) return collapse_sums(structure)
def transform(text): output = StringIO() generate(parse(text), output=output) return output.getvalue()
def get_apps(self, the_makefile, the_dict): input = open(the_makefile).read() output = parse(input) apps = [each for each in output if 'Command' in str(type(each))] otb_apps = [each for each in apps if 'OTB_TEST_APPLICATION' in each.name.upper()] return otb_apps
def get_tests(self, the_makefile, the_dict): input = open(the_makefile).read() output = parse(input) apps = [each for each in output if "Command" in unicode(type(each))] otb_tests = [each for each in apps if "ADD_TEST" in each.name.upper()] return otb_tests
def pip(code=None, args=None, interactive=True): if code or args: interactive = False if interactive: print("=== Welcome to Pip, version %s ===" % VERSION) print("Enter command-line args, terminated by newline (-h for help):") args = input() if args is not None: # Artificial command-line input was provided sys.argv = sys.argv[:1] if type(args) is int: args = str(args) if type(args) is list: # Add list of args to sys.argv, making sure they're all strings sys.argv.extend(map(str, args)) elif type(args) is str: # Parse the fake command-line input, simplistically accepting # single- and double-quoted strings (with no escapes or shell # expansion) quote = None buffer = None for char in args + " ": if char in "'\"": if quote is None: # Open quote quote = char buffer = buffer or "" elif quote == char: # Close quote quote = None else: # Already inside the other type of quote buffer += char elif char == " " and quote is None: if buffer is not None: sys.argv.append(buffer) buffer = None else: buffer = buffer or "" buffer += char argparser = argparse.ArgumentParser() codeSources = argparser.add_mutually_exclusive_group() listFormats = argparser.add_mutually_exclusive_group() argparser.add_argument("-d", "--debug", help="equivalent to -pvw", action="store_true") codeSources.add_argument("-e", "--execute", help="execute the given code") codeSources.add_argument("-f", "--file", help="execute code from the given file") codeSources.add_argument("-i", "--stdin", help="execute code read from stdin", action="store_true") listFormats.add_argument("-l", "--lines", help=("output list items on separate lines, " "concatenated"), action="store_true") listFormats.add_argument("-n", "--newline", help="concatenate lists on newline", action="store_true") listFormats.add_argument("-p", "--repr", help="print lists in repr form", action="store_true") listFormats.add_argument("-P", "--reprlines", help=("output list items on separate lines, " "repr'd"), action="store_true") argparser.add_argument("-r", "--readlines", help="read args from lines of stdin", action="store_true") listFormats.add_argument("-s", "--space", help="concatenate lists on space", action="store_true") listFormats.add_argument("-S", "--spacelines", help=("output list items on separate lines, " "space-concatenated"), action="store_true") argparser.add_argument("-v", "--verbose", help="show extra messages", action="store_true") argparser.add_argument("-V", "--version", help="display version info and quit", action="store_true") argparser.add_argument("-w", "--warnings", help="show nonfatal warning messages", action="store_true") argparser.add_argument("args", help="arguments to main function", nargs="*") options = argparser.parse_args() #!print(options) if options.version: print("Pip %s" % VERSION) return if options.debug: options.warnings = options.verbose = options.repr = True listFormat = ("p" if options.repr else "P" if options.reprlines else "s" if options.space else "S" if options.spacelines else "n" if options.newline else "l" if options.lines else None) if not (code or options.execute or options.file or options.stdin): if interactive: options.stdin = True print("Enter your program, terminated by Ctrl-D or Ctrl-Z:") elif options.args: # Treat first non-option arg as name of code file options.file = options.args.pop(0) else: print("Type {} -h for usage information.".format(sys.argv[0])) return if code: # Code is passed into function program = code + "\n" elif options.execute: # Code is given as command-line argument program = options.execute + "\n" elif options.file: # Get code from specified file if interactive: print("Reading", options.file) try: with open(options.file) as f: program = f.read() + "\n" except: print("Could not read from file", options.file, file=sys.stderr) return elif options.stdin: # Get code from stdin, stopping at EOF program = "\n" try: while True: program += input() + "\n" except EOFError: pass try: tkns = scan(program) except FatalError: print("Fatal error while scanning, execution aborted.", file=sys.stderr) return if options.verbose: print(addSpaces(tkns)) print() try: tree = parse(tkns) except FatalError: print("Fatal error while parsing, execution aborted.", file=sys.stderr) return if options.verbose: pprint.pprint(tree) print() state = ProgramState(listFormat, options.warnings) if options.readlines: args = [] try: while True: args.append(input()) except EOFError: pass else: args = options.args if interactive: print("Executing...") try: state.executeProgram(tree, args) except FatalError: print("Fatal error during execution, program terminated.", file=sys.stderr) except KeyboardInterrupt: print("Program terminated by user.", file=sys.stderr) except RuntimeError as e: # Probably exceeded Python's max recursion depth print("Fatal error:", e, file=sys.stderr)
neighbors_with_movie.append(x) except KeyError: continue bottom = 0 for x in neighbors_with_movie: bottom += abs(tree.w[user][x]) try: predicted = mean + top / bottom except ZeroDivisionError: predicted = None return predicted if __name__ == '__main__': from parsing import parse import pickle import sys test = sys.argv[1] G = pickle.load(open(os.path.join(BASE_DIR, test + ".graph"), "rb")) with open(os.path.join(BASE_DIR, test + "forest"), "rb") as f: component = pickle.load(f) test_rating = parse(os.path.join(BASE_DIR, test + '.test')) base_rating = parse(os.path.join(BASE_DIR, test + '.base')) predictor = Predictor(component, base_rating) test_user = test_rating.keys()[0] test_movie = test_rating[test_user].keys()[0] predictor.predict(test_user, test_movie)
def send_text(message): track_details = parsing.parse(message.text) bot.send_message(message.chat.id, track_details)