def articles_parser_insert_mysql(self): #74218
self.cursor.execute(
"SELECT id, title, content FROM articles where id >= 198886 and id <= 200000"
)
sql = "INSERT INTO articles_parser (id, title_parser_result, content_parser_result) VALUES (%s, %s, %s)"
results = self.cursor.fetchall()
for record in results:
index = record[0]
title = record[1]
content = record[2]
print(index)
print(title, end="\n\n")
print(content)
if title != "":
title_parser_result = parsing.Parser(
re.sub(
r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——！，。？、~@#￥%……&*（）]+",
"", title))
if len(title_parser_result) != 0:
if title_parser_result[0] == "error":
title = HanziConv.toTraditional(title)
title_parser_result = parsing.Parser(
re.sub(
r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——！，。？、~@#￥%……&*（）]+",
"", title))
else:
continue
else:
continue
content = re.sub(r'\、|\，|\。|\?|\？|\;|\；|\:|\~|\：|\⋯|\!', '\n',
content)
content_parser_result = ""
for line in content.split("\n"):
line = re.sub(
r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——！，。？、~@#￥%……&*（）]+",
"", line)
if len(line) >= 4 and '★' not in line and '◆' not in line:
print(line)
parser_result = parsing.Parser(line)
else:
continue
if line == "" or len(
parser_result) is not 1 or parser_result[0] == 'error':
continue
content_parser_result += parser_result[0]
content_parser_result += "@"
time.sleep(self.sleep)
val = (index, title_parser_result[0], content_parser_result)
print(title_parser_result[0], end="\n\n")
print(content_parser_result)
self.cursor.execute(sql, val)
self.db.commit()
self.db.close()
def load(self, cfg):
if cfg.debug_loading:
print("[-vloading] Loading " + repr(self.path) + "...")
if self.path in File._already_loaded:
raise ValueError("cyclic child")
else:
File._already_loaded.append(self.path)
f = open(self.path, "r")
text = f.read()
lexer = lexing.Lexer(text, self.path)
lexer.lex(cfg)
parser = parsing.Parser(lexer)
parser.parse(cfg)
for i in parser.children:
if i.aid == parsing.ASTID.COMMAND:
if i.text == "title":
for j in i.children:
self.title += ast_to_html(cfg, self, j)
elif i.text == "subtitle":
for j in i.children:
self.subtitle += ast_to_html(cfg, self, j)
elif i.text == "author":
for j in i.children:
self.author += ast_to_html(cfg, self, j)
elif i.text == "date":
for j in i.children:
self.date += ast_to_html(cfg, self, j)
elif i.text == "load":
tmp = File(os.path.join(self.get_dir(), i.children[0].text))
if len(self.children) > 0:
self.children[-1].next = tmp
tmp.prev = self.children[-1]
tmp.parent = self
self.children.append(tmp)
elif i.text in ["h1", "h2", "h3", "h4", "h5", "h6"]:
level = int(i.text[1:])
name = ""
for j in i.children:
name += ast_to_html(cfg, self, j)
item = TOCItem(name, level)
self.toc.append(item)
self.content.append(i)
elif i.text == "label":
if len(self.toc) > 0:
File._refs[i.children[0].text] = Reference(self.toc[-1], self)
else:
error.emit_token_error(child.token, "labels can only be placed after headings")
else:
self.content.append(i)
else:
self.content.append(i)
self.is_loaded = True
for i in self.children:
i.load(cfg)
def translateC(line):
parsedLine = parsing.Parser(line)
translatedComp = translator.Translator(parsedLine.comp()).comp()
translatedDest = translator.Translator(parsedLine.dest()).dest()
translatedJump = translator.Translator(parsedLine.jump()).jump()
return '111' + translatedComp + translatedDest + translatedJump
def movies_parser_insert_mysql(self):
self.cursor.execute("SELECT id, storyline FROM movies Where id >= 823")
sql = "INSERT INTO movies_parser (id, storyline_parser_result) VALUES (%s, %s)"
results = self.cursor.fetchall()
for record in results:
movie_id = record[0]
movie_storyline = record[1]
print(movie_id)
print(movie_storyline)
movie_storyline = re.sub(r'\、|\，|\。|\?|\？|\;|\；|\:|\~|\：|\⋯|\!', '\n', movie_storyline)
storyline_parser_result = ""
for line in movie_storyline.split("\n"):
line = re.sub(r"[\s+\.\【\】\‧\!\/_,$%^*(+\"\']+|[+——！，。？、~@#￥%……&*（）]+", "", line)
if len(line) >= 4 and '★' not in line and '◆' not in line:
print(line)
parser_result = parsing.Parser(line)
else:
continue
if line == "" or len(parser_result) is not 1 or parser_result[0] == 'error':
continue
storyline_parser_result += parser_result[0]
storyline_parser_result += "@"
time.sleep(self.sleep)
val = (movie_id, storyline_parser_result)
print(storyline_parser_result)
self.cursor.execute(sql, val)
self.db.commit()
self.db.close()
def __init__(self, messagesFile="./rp/messages.en-gb.xml"):
self.parser = parsing.Parser()
self.messages = messages.Messages(messagesFile)
self.integerAllowedCharacters = "0123456789+- "
self.nonNegativeIntegerAllowedCharacters = "0123456789+ "
self.decimalAllowedCharacters = "0123456789.+- "
def __init__(self):
self.parser = parsing.Parser(self.grammar, self.lexer)
self.import_stack = []
self.imported = set()
self.cache_hits = 0
self.cache_misses = 0
self.parsing_cache = {}
self.cwd = "."
self.newest_source_time = float("-inf")
def treat_entry(filename, facts_input=None, mode='backward_chaining'):
parse = parsing.Parser(filename)
rules_lst, facts_lst, queries_lst = parse.parse_file()
if mode == 'backward_chaining':
return backward_chaining_solve(rules_lst, facts_lst, queries_lst,
facts_input)
else:
return resolution_solver.resolution_solve(rules_lst, facts_lst,
queries_lst, facts_input)
def fill_hydrogens(t):
"""Takes a flattened list of Atom objects (from the function flatten()) and fills in all the missing hydrogen Atoms such that each Atom object has four bonds.
t: list of Atom objects
Returns: list of Atom objects.
"""
p = parsing.Parser()
res = fill(t, p)
return res
def connect_device(port, baud, packet_queue, broadcast, relay_queue, relay):
parsing_queue = Queue()
if relay:
client_thread = client.Client(parsing_queue, port, baud, relay_queue)
else:
client_thread = client.Client(parsing_queue, port, baud)
parser_thread = parsing.Parser(parsing_queue, packet_queue, broadcast,
relay_queue)
client_thread.start()
parser_thread.start()
def __init__(self,
n_sp,
generations,
neighbourhood,
de_F,
de_CR,
de_SIG,
crossover_mode,
path_to_file,
const_mode=None,
problem=zdt3.ZDT3(),
weights=False):
self.n_sp = n_sp
self.problem = problem
self.generations = generations
self.de_F = de_F
self.de_CR = de_CR
self.de_SIG = de_SIG
self.const = const_mode
self.crossover = crossover_mode
self.weights = weights if weights else np.array(
[1 for _ in range(problem.n_con)])
self.EP = list()
self.lambda_vectors = [
np.array([n / (self.n_sp - 1), 1 - (n / (self.n_sp - 1))])
for n in range(self.n_sp)
]
self.neighbours = {}
for i in range(self.n_sp):
self.neighbours[i] = self.get_neighbours(self.lambda_vectors[i],
list(range(self.n_sp)),
self.lambda_vectors,
neighbourhood)
self.population = [
np.random.uniform(self.problem.min_real, self.problem.max_real)
for _ in range(self.n_sp)
]
performances = [self.problem.func(s) for s in self.population]
self.z_star = np.array([
min([ind[o] for ind in performances])
for o in range(self.problem.n_obj)
])
self.initialize_EP(performances)
self.parser = parsing.Parser(path_to_file)
def run(filename):
with open(filename) as f:
source = f.read()
parser = parsing.Parser(source, filename=filename)
ast = parser.parse_program()
#print(ast.show())
typechecker_ = typechecker.TypeChecker()
checked_ast = typechecker_.check_program(ast)
#print(checked_ast.show())
evaluator = evaluator_bfs.Evaluator()
results = evaluator.eval_program(checked_ast, strategy='strong')
for result in results:
print(result.show())
input(" ; ")
print("done.")
def get_articles_range(a_range, num_articles, **kwargs):
"""
This function serves as the main function.
1. Calls downloader.Updater.update() and saves new articles from the web into the database
2. Calls parsing.Parser.parse_date_range() that collects the latest news from the database and parses it into JSON,
that is then read by the JavaScript Cesium application and displayed in the browser.
:param a_range: Days before next midnight to parse.
:param num_articles: Maximum number of articles to be parsed.
:param kwargs: Other arguments
:return: Returns list of Updater and Parser instances.
"""
global downloader
global parsing
logger.info('')
# For handling test case
if "test" in kwargs:
downloader = kwargs.get("downloader")
parsing = kwargs.get("parsing")
print("Getting article range...")
u = downloader.Updater()
u.update(num_articles)
u.save_to_database()
p = parsing.Parser()
p.parse_date_range(timedelta=a_range)
if "ftp_host" in kwargs and "ftp_username" in kwargs and "ftp_password" in kwargs:
if kwargs["ftp_host"] and kwargs["ftp_username"] and kwargs[
"ftp_password"]:
ftp_upload.update_all_files(ftp_host=kwargs["ftp_host"],
ftp_port=kwargs["ftp_port"],
ftp_username=kwargs["ftp_username"],
ftp_password=kwargs["ftp_password"],
ftp_rootdir=kwargs["ftp_rootdir"])
utils.delete_location_logs()
return [u, p]
def run(self, code):
#First we scan for tokens
scanner = lexer.Scanner(code.upper())
tokens = scanner.scanTokens()
#We use None as a way to return Errors
if (tokens == None):
print("Lexical error encountered. Not executing")
return
#Next we parse the tokens we've generated into trees
parser = parsing.Parser(tokens)
trees = parser.parse()
if (trees == None):
print("Parsing error encountered. Not executing")
return
#The next section forms a middle end, that changes the not so assembly code so far
#into proper assembly code that the actual transputer would be able to handle
#We should check for any prefix errors that may come up and be undetecable
#at run time after we expand out illegal operands in the expansion step
prefixChanger = prefixChecking.PrefixPreparer(trees)
trees = prefixChanger.prepare()
if (trees == None):
print("Illegal prefix sequence encountered. Not executing")
return
#Then we use an expander to expand out double length instructions
#And operands longer than 4 bits
#We also check if there is any overflow when expanding out jumps
expander = expanding.Expander(trees)
trees = expander.expand()
if (trees == None):
print("Ran into a problem while correcting Jumps. Not executing")
return
#After this we're ready to interpret!
self.vm.run(trees)
class Forecaster(object):
regions = json.loads(open('cities.txt', 'r').read())
message_parser = parsing.Parser()
translator = translation.Translator()
def __init__(self):
super().__init__()
self.__header = {'X-Yandex-API-Key': config.WEATHER_API}
self.__req = 'https://api.weather.yandex.ru/v1/forecast?geoid={}&limit={}?l10n={}'
def forecast(self, message):
request = Forecaster.message_parser.parse(message)
if request[0] == -1:
return BAD_QUERY
city, shift, period = request
city_eng = Forecaster.translator.translate_to_en(city)[0].lower()
if city_eng == -1:
return BAD_QUERY
forecast_geoid = Forecaster.regions.get(city_eng, None)
if forecast_geoid is None:
return BAD_QUERY
request = self.__req.format(forecast_geoid, min(10, max(shift, 2)),
'true')
request = requests.get(request, headers=self.__header)
response = json.loads(request.content.decode('utf-8'))
forecast = response['fact']
if shift > 0:
forecast = response['forecasts'][min(shift, 10)]
forecast = forecast['parts'][period]
img_weather, str_message = parse_forecast(forecast)
return (img_weather, city_eng, str_message)
already_exists = False
for confirmed_clause in self.confirmed_clauses:
if (clause.negative_facts == confirmed_clause.negative_facts
and clause.positive_facts
== confirmed_clause.positive_facts):
already_exists = True
if not already_exists:
self.confirmed_clauses.add(clause)
def get_facts_from_confirmed_clauses(self):
confirmed_facts = set()
for clause in self.confirmed_clauses:
clause_facts = (clause.positive_facts | clause.negative_facts)
if len(clause_facts) == 1:
fact_content = clause_facts.pop()
confirmed_facts.add(fact_content)
return confirmed_facts
if __name__ == '__main__':
try:
if len(sys.argv) != 2:
raise Exception(parsing.INPUT_ERROR)
file = sys.argv[1]
parse = parsing.Parser(file)
rules_list, facts_list, queries_list = parse.parse_file()
G = Graph(rules_list, facts_list, queries_list)
except Exception as e:
print(e)
def printDatabase(self, database):
self.printToFile("========== DATABASE DUMP ==========")
for table in database.tables:
self.printTable(table)
self.printToFile("")
def printTable(self, table):
self.printToFile("======= TABLE: %s.%s" % (table.schema, table.name))
for record in table.records:
for column, value in record.data.items():
self.printToFile("%s: %s" % (column, value), 1)
self.printToFile("")
if __name__ == '__main__':
p = parsing.Parser()
verbosity = 0 if p.args.verbose is None else p.args.verbose
printer = Printer(verbosity, p.args.outfile)
try:
cookies = p.args.cookie
except AttributeError:
cookies = None
if p.args.shell is not None:
stringExecutor = concurrent.futures.ThreadPoolExecutor(
max_workers=max(math.floor(1 / 9 * p.args.max_threads), 1))
charExecutor = concurrent.futures.ThreadPoolExecutor(
max_workers=max(math.floor(8 / 9 * p.args.max_threads), 1))
try:
print("Loading injector...")
else:
facts_list = list(new_facts)
if set(facts_list) <= set(parsing.ALLOWED_FACTS):
new_facts = list(set(facts_list))
input_error = False
else:
input_error = True
print("\nWrong input, please try again.")
if __name__ == '__main__':
try:
options = get_args()
filename = options.filename
parse = parsing.Parser(filename)
rules_lst, facts_lst, queries_lst = parse.parse_file()
if (options.complete or options.proof
) and options.resolution_mode == "backward_chaining":
complete = check_if_rules_horn_clauses(rules_lst)
if not complete:
print(
"Be aware: some rules are not Horn Clauses. The backward_chaining algorithm is not complete."
)
if options.proof:
print(
"The proof may therefore be incomplete. It should work most of the time for simple proofs.\n"
)
else:
print()
if words[1] in important_parts:
d.set_speach_part(HEAD_WORD)
d.combine_bags(HEAD_WORD, word)
counter += 1
### Load FREQUENCY ###
frequency_file = "./data/de_full_opensubtitle.txt"
not_found_file = "./data/not_found_words.txt"
d.load_frequencies(frequency_file, not_found_file)
### Test ###
import parsing
pars = parsing.Parser(d)
pars.parse("./data/text.txt", "./data/test.txt")
#top 5000 words
with open("./data/5000.txt", 'w') as f:
ordered_list = d.Bags.get_top(5000)
counter = 0
for bag in ordered_list:
max_freq = -1
most_freq_word = None
for w in bag:
freq = d[w][2]
if freq > max_freq:
def voiceLoop():
global restartLoop
config = settings.loadConfig()
AUDIO_TIMEOUT = config["SETTINGS"]["TIMEOUT"] # length of pause marking end of command
with open('command_set.txt', 'r') as myfile:
str_command_set = myfile.read()
command_set = str_command_set.split('\n')
in_debug_mode = False
if os.path.exists('DEBUG_FLAG'):
in_debug_mode = True
log.info("debug mode activated")
opened = False
while not opened:
try:
pipe = win32file.CreateFile(
r'\\.\pipe\named_pipe',
win32file.GENERIC_READ | win32file.GENERIC_WRITE,
win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_READ,
None, win32file.OPEN_EXISTING, 0, None)
opened = True
except Exception as e:
log.error("HELLO WORLD")
log.error(str(e))
log.error(traceback.format_exc())
time.sleep(1)
time.sleep(1)
else:
log.info("voice mode activated")
p = parsing.Parser()
with mic as source:
#Automatically adjust for ambient noise instead of calling calibrate
r.dynamic_energy_threshold = True
r.pause_threshold = AUDIO_TIMEOUT
while True:
try:
gui.ready()
raw_command = ''
if in_debug_mode:
message = win32file.ReadFile(pipe, 4096)
log.debug('pipe message: ', message[1].decode())
raw_command = message[1].decode()
else:
audio = r.listen(source)
# recognize speech using Google Cloud Speech API
log.debug("Pre recognize")
raw_command = recognize(audio, command_set)
gui.processing()
cmdPromptHandle = None
if in_debug_mode and not os.path.exists('BATCH_FLAG'):
keyboard.press_and_release("alt+tab")
time.sleep(1) # give OS time to alt-tab
if raw_command == -1:
raise ValueError("Failed to recognize speech")
else:
p.parse(raw_command)
if os.path.exists('BATCH_FLAG'):
# send an ACK to tell them we're ready for more input
win32file.WriteFile(pipe, 'ACK'.encode())
elif in_debug_mode:
time.sleep(1) # give the user time to see the result
commands.exeFocus(['CMD'], GlobalMode.NAVIGATE)
gui.updateCommands(raw_command)
except Exception as e:
log.error(str(e))
log.error(traceback.format_exc())
errorMsg = None
try:
log.debug('type(raw_command) = {}'.format(type(raw_command)))
# this looks incredibly dumb, but there's a reason: we really
# just want to check if raw_command is an integer. This could be
# refactored but #timepressure
if raw_command == -1:
errorMsg = '(google api failed!)'
else:
errorMsg = '(google api failed!)'
except ValueError:
errorMsg = 'error: ' + str(raw_command)
gui.updateCommands(errorMsg)
gui.showError("Error parsing\nTry again.")
if p.mode == GlobalMode.FOLLOW:
continue
gui.setMode(p.mode)
def main():
parse_obj = parsing.Parser()
args = parse_obj.parse_arguments()
print(args)
return 0
def __init__(self, *args, **kwargs):
Humanoid.__init__(self, *args, **kwargs)
self.traits.add(trait.hero())
self.ai = parsing.Parser(self)
self.visited_locations = set()
self.combat_skill = 100