def articles_parser_insert_mysql(self): #74218 self.cursor.execute( "SELECT id, title, content FROM articles where id >= 198886 and id <= 200000" ) sql = "INSERT INTO articles_parser (id, title_parser_result, content_parser_result) VALUES (%s, %s, %s)" results = self.cursor.fetchall() for record in results: index = record[0] title = record[1] content = record[2] print(index) print(title, end="\n\n") print(content) if title != "": title_parser_result = parsing.Parser( re.sub( r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——!,。?、~@#¥%……&*()]+", "", title)) if len(title_parser_result) != 0: if title_parser_result[0] == "error": title = HanziConv.toTraditional(title) title_parser_result = parsing.Parser( re.sub( r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——!,。?、~@#¥%……&*()]+", "", title)) else: continue else: continue content = re.sub(r'\、|\,|\。|\?|\?|\;|\;|\:|\~|\:|\⋯|\!', '\n', content) content_parser_result = "" for line in content.split("\n"): line = re.sub( r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——!,。?、~@#¥%……&*()]+", "", line) if len(line) >= 4 and '★' not in line and '◆' not in line: print(line) parser_result = parsing.Parser(line) else: continue if line == "" or len( parser_result) is not 1 or parser_result[0] == 'error': continue content_parser_result += parser_result[0] content_parser_result += "@" time.sleep(self.sleep) val = (index, title_parser_result[0], content_parser_result) print(title_parser_result[0], end="\n\n") print(content_parser_result) self.cursor.execute(sql, val) self.db.commit() self.db.close()
def load(self, cfg): if cfg.debug_loading: print("[-vloading] Loading " + repr(self.path) + "...") if self.path in File._already_loaded: raise ValueError("cyclic child") else: File._already_loaded.append(self.path) f = open(self.path, "r") text = f.read() lexer = lexing.Lexer(text, self.path) lexer.lex(cfg) parser = parsing.Parser(lexer) parser.parse(cfg) for i in parser.children: if i.aid == parsing.ASTID.COMMAND: if i.text == "title": for j in i.children: self.title += ast_to_html(cfg, self, j) elif i.text == "subtitle": for j in i.children: self.subtitle += ast_to_html(cfg, self, j) elif i.text == "author": for j in i.children: self.author += ast_to_html(cfg, self, j) elif i.text == "date": for j in i.children: self.date += ast_to_html(cfg, self, j) elif i.text == "load": tmp = File(os.path.join(self.get_dir(), i.children[0].text)) if len(self.children) > 0: self.children[-1].next = tmp tmp.prev = self.children[-1] tmp.parent = self self.children.append(tmp) elif i.text in ["h1", "h2", "h3", "h4", "h5", "h6"]: level = int(i.text[1:]) name = "" for j in i.children: name += ast_to_html(cfg, self, j) item = TOCItem(name, level) self.toc.append(item) self.content.append(i) elif i.text == "label": if len(self.toc) > 0: File._refs[i.children[0].text] = Reference(self.toc[-1], self) else: error.emit_token_error(child.token, "labels can only be placed after headings") else: self.content.append(i) else: self.content.append(i) self.is_loaded = True for i in self.children: i.load(cfg)
def translateC(line): parsedLine = parsing.Parser(line) translatedComp = translator.Translator(parsedLine.comp()).comp() translatedDest = translator.Translator(parsedLine.dest()).dest() translatedJump = translator.Translator(parsedLine.jump()).jump() return '111' + translatedComp + translatedDest + translatedJump
def movies_parser_insert_mysql(self): self.cursor.execute("SELECT id, storyline FROM movies Where id >= 823") sql = "INSERT INTO movies_parser (id, storyline_parser_result) VALUES (%s, %s)" results = self.cursor.fetchall() for record in results: movie_id = record[0] movie_storyline = record[1] print(movie_id) print(movie_storyline) movie_storyline = re.sub(r'\、|\,|\。|\?|\?|\;|\;|\:|\~|\:|\⋯|\!', '\n', movie_storyline) storyline_parser_result = "" for line in movie_storyline.split("\n"): line = re.sub(r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——!,。?、~@#¥%……&*()]+", "", line) if len(line) >= 4 and '★' not in line and '◆' not in line: print(line) parser_result = parsing.Parser(line) else: continue if line == "" or len(parser_result) is not 1 or parser_result[0] == 'error': continue storyline_parser_result += parser_result[0] storyline_parser_result += "@" time.sleep(self.sleep) val = (movie_id, storyline_parser_result) print(storyline_parser_result) self.cursor.execute(sql, val) self.db.commit() self.db.close()
def __init__(self, messagesFile="./rp/messages.en-gb.xml"): self.parser = parsing.Parser() self.messages = messages.Messages(messagesFile) self.integerAllowedCharacters = "0123456789+- " self.nonNegativeIntegerAllowedCharacters = "0123456789+ " self.decimalAllowedCharacters = "0123456789.+- "
def __init__(self): self.parser = parsing.Parser(self.grammar, self.lexer) self.import_stack = [] self.imported = set() self.cache_hits = 0 self.cache_misses = 0 self.parsing_cache = {} self.cwd = "." self.newest_source_time = float("-inf")
def treat_entry(filename, facts_input=None, mode='backward_chaining'): parse = parsing.Parser(filename) rules_lst, facts_lst, queries_lst = parse.parse_file() if mode == 'backward_chaining': return backward_chaining_solve(rules_lst, facts_lst, queries_lst, facts_input) else: return resolution_solver.resolution_solve(rules_lst, facts_lst, queries_lst, facts_input)
def fill_hydrogens(t): """Takes a flattened list of Atom objects (from the function flatten()) and fills in all the missing hydrogen Atoms such that each Atom object has four bonds. t: list of Atom objects Returns: list of Atom objects. """ p = parsing.Parser() res = fill(t, p) return res
def connect_device(port, baud, packet_queue, broadcast, relay_queue, relay): parsing_queue = Queue() if relay: client_thread = client.Client(parsing_queue, port, baud, relay_queue) else: client_thread = client.Client(parsing_queue, port, baud) parser_thread = parsing.Parser(parsing_queue, packet_queue, broadcast, relay_queue) client_thread.start() parser_thread.start()
def __init__(self, n_sp, generations, neighbourhood, de_F, de_CR, de_SIG, crossover_mode, path_to_file, const_mode=None, problem=zdt3.ZDT3(), weights=False): self.n_sp = n_sp self.problem = problem self.generations = generations self.de_F = de_F self.de_CR = de_CR self.de_SIG = de_SIG self.const = const_mode self.crossover = crossover_mode self.weights = weights if weights else np.array( [1 for _ in range(problem.n_con)]) self.EP = list() self.lambda_vectors = [ np.array([n / (self.n_sp - 1), 1 - (n / (self.n_sp - 1))]) for n in range(self.n_sp) ] self.neighbours = {} for i in range(self.n_sp): self.neighbours[i] = self.get_neighbours(self.lambda_vectors[i], list(range(self.n_sp)), self.lambda_vectors, neighbourhood) self.population = [ np.random.uniform(self.problem.min_real, self.problem.max_real) for _ in range(self.n_sp) ] performances = [self.problem.func(s) for s in self.population] self.z_star = np.array([ min([ind[o] for ind in performances]) for o in range(self.problem.n_obj) ]) self.initialize_EP(performances) self.parser = parsing.Parser(path_to_file)
def run(filename): with open(filename) as f: source = f.read() parser = parsing.Parser(source, filename=filename) ast = parser.parse_program() #print(ast.show()) typechecker_ = typechecker.TypeChecker() checked_ast = typechecker_.check_program(ast) #print(checked_ast.show()) evaluator = evaluator_bfs.Evaluator() results = evaluator.eval_program(checked_ast, strategy='strong') for result in results: print(result.show()) input(" ; ") print("done.")
def get_articles_range(a_range, num_articles, **kwargs): """ This function serves as the main function. 1. Calls downloader.Updater.update() and saves new articles from the web into the database 2. Calls parsing.Parser.parse_date_range() that collects the latest news from the database and parses it into JSON, that is then read by the JavaScript Cesium application and displayed in the browser. :param a_range: Days before next midnight to parse. :param num_articles: Maximum number of articles to be parsed. :param kwargs: Other arguments :return: Returns list of Updater and Parser instances. """ global downloader global parsing logger.info('') # For handling test case if "test" in kwargs: downloader = kwargs.get("downloader") parsing = kwargs.get("parsing") print("Getting article range...") u = downloader.Updater() u.update(num_articles) u.save_to_database() p = parsing.Parser() p.parse_date_range(timedelta=a_range) if "ftp_host" in kwargs and "ftp_username" in kwargs and "ftp_password" in kwargs: if kwargs["ftp_host"] and kwargs["ftp_username"] and kwargs[ "ftp_password"]: ftp_upload.update_all_files(ftp_host=kwargs["ftp_host"], ftp_port=kwargs["ftp_port"], ftp_username=kwargs["ftp_username"], ftp_password=kwargs["ftp_password"], ftp_rootdir=kwargs["ftp_rootdir"]) utils.delete_location_logs() return [u, p]
def run(self, code): #First we scan for tokens scanner = lexer.Scanner(code.upper()) tokens = scanner.scanTokens() #We use None as a way to return Errors if (tokens == None): print("Lexical error encountered. Not executing") return #Next we parse the tokens we've generated into trees parser = parsing.Parser(tokens) trees = parser.parse() if (trees == None): print("Parsing error encountered. Not executing") return #The next section forms a middle end, that changes the not so assembly code so far #into proper assembly code that the actual transputer would be able to handle #We should check for any prefix errors that may come up and be undetecable #at run time after we expand out illegal operands in the expansion step prefixChanger = prefixChecking.PrefixPreparer(trees) trees = prefixChanger.prepare() if (trees == None): print("Illegal prefix sequence encountered. Not executing") return #Then we use an expander to expand out double length instructions #And operands longer than 4 bits #We also check if there is any overflow when expanding out jumps expander = expanding.Expander(trees) trees = expander.expand() if (trees == None): print("Ran into a problem while correcting Jumps. Not executing") return #After this we're ready to interpret! self.vm.run(trees)
class Forecaster(object): regions = json.loads(open('cities.txt', 'r').read()) message_parser = parsing.Parser() translator = translation.Translator() def __init__(self): super().__init__() self.__header = {'X-Yandex-API-Key': config.WEATHER_API} self.__req = 'https://api.weather.yandex.ru/v1/forecast?geoid={}&limit={}?l10n={}' def forecast(self, message): request = Forecaster.message_parser.parse(message) if request[0] == -1: return BAD_QUERY city, shift, period = request city_eng = Forecaster.translator.translate_to_en(city)[0].lower() if city_eng == -1: return BAD_QUERY forecast_geoid = Forecaster.regions.get(city_eng, None) if forecast_geoid is None: return BAD_QUERY request = self.__req.format(forecast_geoid, min(10, max(shift, 2)), 'true') request = requests.get(request, headers=self.__header) response = json.loads(request.content.decode('utf-8')) forecast = response['fact'] if shift > 0: forecast = response['forecasts'][min(shift, 10)] forecast = forecast['parts'][period] img_weather, str_message = parse_forecast(forecast) return (img_weather, city_eng, str_message)
already_exists = False for confirmed_clause in self.confirmed_clauses: if (clause.negative_facts == confirmed_clause.negative_facts and clause.positive_facts == confirmed_clause.positive_facts): already_exists = True if not already_exists: self.confirmed_clauses.add(clause) def get_facts_from_confirmed_clauses(self): confirmed_facts = set() for clause in self.confirmed_clauses: clause_facts = (clause.positive_facts | clause.negative_facts) if len(clause_facts) == 1: fact_content = clause_facts.pop() confirmed_facts.add(fact_content) return confirmed_facts if __name__ == '__main__': try: if len(sys.argv) != 2: raise Exception(parsing.INPUT_ERROR) file = sys.argv[1] parse = parsing.Parser(file) rules_list, facts_list, queries_list = parse.parse_file() G = Graph(rules_list, facts_list, queries_list) except Exception as e: print(e)
def printDatabase(self, database): self.printToFile("========== DATABASE DUMP ==========") for table in database.tables: self.printTable(table) self.printToFile("") def printTable(self, table): self.printToFile("======= TABLE: %s.%s" % (table.schema, table.name)) for record in table.records: for column, value in record.data.items(): self.printToFile("%s: %s" % (column, value), 1) self.printToFile("") if __name__ == '__main__': p = parsing.Parser() verbosity = 0 if p.args.verbose is None else p.args.verbose printer = Printer(verbosity, p.args.outfile) try: cookies = p.args.cookie except AttributeError: cookies = None if p.args.shell is not None: stringExecutor = concurrent.futures.ThreadPoolExecutor( max_workers=max(math.floor(1 / 9 * p.args.max_threads), 1)) charExecutor = concurrent.futures.ThreadPoolExecutor( max_workers=max(math.floor(8 / 9 * p.args.max_threads), 1)) try: print("Loading injector...")
else: facts_list = list(new_facts) if set(facts_list) <= set(parsing.ALLOWED_FACTS): new_facts = list(set(facts_list)) input_error = False else: input_error = True print("\nWrong input, please try again.") if __name__ == '__main__': try: options = get_args() filename = options.filename parse = parsing.Parser(filename) rules_lst, facts_lst, queries_lst = parse.parse_file() if (options.complete or options.proof ) and options.resolution_mode == "backward_chaining": complete = check_if_rules_horn_clauses(rules_lst) if not complete: print( "Be aware: some rules are not Horn Clauses. The backward_chaining algorithm is not complete." ) if options.proof: print( "The proof may therefore be incomplete. It should work most of the time for simple proofs.\n" ) else: print()
if words[1] in important_parts: d.set_speach_part(HEAD_WORD) d.combine_bags(HEAD_WORD, word) counter += 1 ### Load FREQUENCY ### frequency_file = "./data/de_full_opensubtitle.txt" not_found_file = "./data/not_found_words.txt" d.load_frequencies(frequency_file, not_found_file) ### Test ### import parsing pars = parsing.Parser(d) pars.parse("./data/text.txt", "./data/test.txt") #top 5000 words with open("./data/5000.txt", 'w') as f: ordered_list = d.Bags.get_top(5000) counter = 0 for bag in ordered_list: max_freq = -1 most_freq_word = None for w in bag: freq = d[w][2] if freq > max_freq:
def voiceLoop(): global restartLoop config = settings.loadConfig() AUDIO_TIMEOUT = config["SETTINGS"]["TIMEOUT"] # length of pause marking end of command with open('command_set.txt', 'r') as myfile: str_command_set = myfile.read() command_set = str_command_set.split('\n') in_debug_mode = False if os.path.exists('DEBUG_FLAG'): in_debug_mode = True log.info("debug mode activated") opened = False while not opened: try: pipe = win32file.CreateFile( r'\\.\pipe\named_pipe', win32file.GENERIC_READ | win32file.GENERIC_WRITE, win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_READ, None, win32file.OPEN_EXISTING, 0, None) opened = True except Exception as e: log.error("HELLO WORLD") log.error(str(e)) log.error(traceback.format_exc()) time.sleep(1) time.sleep(1) else: log.info("voice mode activated") p = parsing.Parser() with mic as source: #Automatically adjust for ambient noise instead of calling calibrate r.dynamic_energy_threshold = True r.pause_threshold = AUDIO_TIMEOUT while True: try: gui.ready() raw_command = '' if in_debug_mode: message = win32file.ReadFile(pipe, 4096) log.debug('pipe message: ', message[1].decode()) raw_command = message[1].decode() else: audio = r.listen(source) # recognize speech using Google Cloud Speech API log.debug("Pre recognize") raw_command = recognize(audio, command_set) gui.processing() cmdPromptHandle = None if in_debug_mode and not os.path.exists('BATCH_FLAG'): keyboard.press_and_release("alt+tab") time.sleep(1) # give OS time to alt-tab if raw_command == -1: raise ValueError("Failed to recognize speech") else: p.parse(raw_command) if os.path.exists('BATCH_FLAG'): # send an ACK to tell them we're ready for more input win32file.WriteFile(pipe, 'ACK'.encode()) elif in_debug_mode: time.sleep(1) # give the user time to see the result commands.exeFocus(['CMD'], GlobalMode.NAVIGATE) gui.updateCommands(raw_command) except Exception as e: log.error(str(e)) log.error(traceback.format_exc()) errorMsg = None try: log.debug('type(raw_command) = {}'.format(type(raw_command))) # this looks incredibly dumb, but there's a reason: we really # just want to check if raw_command is an integer. This could be # refactored but #timepressure if raw_command == -1: errorMsg = '(google api failed!)' else: errorMsg = '(google api failed!)' except ValueError: errorMsg = 'error: ' + str(raw_command) gui.updateCommands(errorMsg) gui.showError("Error parsing\nTry again.") if p.mode == GlobalMode.FOLLOW: continue gui.setMode(p.mode)
def main(): parse_obj = parsing.Parser() args = parse_obj.parse_arguments() print(args) return 0
def __init__(self, *args, **kwargs): Humanoid.__init__(self, *args, **kwargs) self.traits.add(trait.hero()) self.ai = parsing.Parser(self) self.visited_locations = set() self.combat_skill = 100