def print_parsed(parsed: dict or list, depth: int = 0):
"""
Print The result of methods @parse and @relevant_parse (so a parsed html)
:param parsed: A parsed html
:param depth: Current depth to build a pretty tree
"""
space = ' ' * depth
if type(parsed) == dict:
print(space + '{')
for key, value in parsed.items():
if key == 'children':
HtmlParser.print_parsed(value, depth + 1)
elif is_listable(value):
print((space + ' ') + str(key) + ':')
HtmlParser.print_parsed(value, depth + 2)
# print((space+' ') + str(key) + ':')
# subspace = ' ' * (depth+1)
# for el in dict:
# if (is_listable(el)):
else:
print((space + ' ') + str(key) + ': ' + str(value))
print(space + '}')
elif type(parsed) == list:
for value in parsed:
HtmlParser.print_parsed(value, depth + 1)
else:
Log.error(str(parsed) + ' is not a valid parsed content!')
def session_get(self, session, params=None):
"""
:type session: django.contrib.sessions.backends.db.SessionStore
:type params: dict
:rtype: dict
"""
if params is None:
params = {}
session_params = session.get(self.name)
if session_params is None:
session_params = dict()
return_params = session_params
for key, values in params.items():
session_value = session_params.get(key)
if is_listable(values):
return_values = dict()
for value in values:
return_values[value] = (value == session_value)
else:
return_values = {{values: (session_value is not None)}}
return_params[key] = return_values
return return_params
def start(self, target, args, asynchronous, cpu):
self.tasks = []
def task_target(*arguments):
result = None
if self.tasks_type == MultiTask.MULTI_PROCESSING:
curr_task = multiprocessing.process.current_process()
Log.info(self.tag + 'started (PID=' + str(curr_task.pid) + ')')
else:
curr_task = threading.current_thread()
Log.info(self.tag + 'started')
if target is not None:
result = target(*arguments)
if result is not None:
Log.success("Result: " + str(result))
# Scrivo il risultato nel file
Log.info('Writing result in ' + str(self.resfile))
storage.overwrite_file(str(result), self.resfile) # TODO: dump result as object with "pickle"
# Termino tutti gli altri threads/processi
if self.tasks_type == MultiTask.MULTI_PROCESSING:
Log.info('Killing other processes')
running_pids = MultiTask.get_pids_from_file(self.pidfile)
for pid in running_pids:
pid = int(pid)
if pid == curr_task.pid:
continue
try:
os.kill(pid, signal.SIGKILL)
Log.info('Process ' + str(pid) + ' killed!')
except Exception as e:
Log.error(str(e))
Log.info(self.tag + 'end')
else:
Log.info('Ignoring other threads')
# Killa se stesso
pid = multiprocessing.process.current_process().pid
Log.info(self.tag + 'end')
os.kill(pid, signal.SIGKILL)
for i in range(0, cpu):
task_args = ()
for arg in args:
Log.info('Argument type: ' + str(type(arg)))
if is_listable(arg):
# Divido gli elementi in 1/cpu parti
p_list_len = (len(arg) / cpu) + (len(arg) % cpu)
if type(arg) == dict:
iterator = iter(arg.items())
task_args += (
dict(itertools.islice(iterator, int((i * p_list_len)), int((i + 1) * p_list_len))),
)
else:
task_args += (arg[int((i * p_list_len)):int(((i + 1) * p_list_len))],)
else:
task_args += (arg,)
task = self.Multitask(target=task_target, args=task_args)
self.tasks.append(task)
if self.tasks_type == MultiTask.MULTI_PROCESSING:
pids = []
signal.signal(signal.SIGCHLD, signal.SIG_IGN) # Ignore child exit status
for task in self.tasks:
task.start()
# noinspection PyUnresolvedReferences
pids.append(task.pid)
storage.overwrite_file(str(pids).strip('[]'), self.pidfile)
else:
for task in self.tasks:
task.start()
if not asynchronous:
# Attende la fine dell'esecuzione di tutti i tasks
for task in self.tasks:
task.join()
Log.info('Task ' + str(task.name) + ' joined')
Log.info('Reading result in ' + str(self.resfile))
# Prendo il risultato dal file
res = storage.read_file(self.resfile) # TODO: load result as object with "pickle"
# Elimino l'eventuale file con i pid
storage.delete(self.pidfile)
# Elimino il file con il risultato
storage.delete(self.resfile)
Log.success('MultiTask -> result: ' + str(res))
return res
return None