def run_tasks(self, queue, parallel=True, show_progress=True):
'''this run_tasks function takes a list of Task objects, each
potentially a different kind of task, and extracts the parameters
with task.export_params(), and the running function with
task.export_func(), and hands these over to the multiprocessing
worker. It's up to the Task to return some correct function
from it's set of task functions that correspond with the variables.
Examples
========
funcs
{'task-reddit-hpc': <function watchme.watchers.urls.tasks.get_task>}
tasks
{'task-reddit-hpc': [('url', 'https://www.reddit.com/r/hpc'),
('active', 'true'),
('type', 'urls')]}
'''
if parallel is True:
return self._run_parallel(queue, show_progress)
# Otherwise, run in serial
results = {}
# Progressbar
total = len(queue)
progress = 1
for task in queue:
prefix = "[%s:%s/%s]" % (task.name, progress, total)
if show_progress is True:
bot.show_progress(progress, total, length=35, prefix=prefix)
else:
bot.info('Running %s' % prefix)
results[task.name] = task.run()
progress += 1
return results
def run(self, funcs, tasks):
'''run will send a list of tasks, a tuple with arguments, through a function.
the arguments should be ordered correctly.
Parameters
==========
funcs: the functions to run with multiprocessing.pool, a dictionary
with lookup by the task name
tasks: a dict of tasks, each task name (key) with a
tuple of arguments to process
'''
# Number of tasks must == number of functions
assert (len(funcs) == len(tasks))
# Keep track of some progress for the user
progress = 1
total = len(tasks)
# if we don't have tasks, don't run
if len(tasks) == 0:
return
# results will also have the same key to look up
finished = dict()
results = []
try:
prefix = "[%s/%s]" % (progress, total)
if self.show_progress:
bot.show_progress(0, total, length=35, prefix=prefix)
pool = multiprocessing.Pool(self.workers, init_worker)
self.start()
for key, params in tasks.items():
func = funcs[key]
if not self.show_progress:
bot.info('Processing task %s:%s' % (key, params))
result = pool.apply_async(multi_wrapper,
multi_package(func, [params]))
# Store the key with the result
results.append((key, result))
while len(results) > 0:
pair = results.pop()
key, result = pair
result.wait()
if self.show_progress:
bot.show_progress(progress,
total,
length=35,
prefix=prefix)
progress += 1
prefix = "[%s/%s]" % (progress, total)
finished[key] = result.get()
self.end()
pool.close()
pool.join()
except (KeyboardInterrupt, SystemExit):
bot.error("Keyboard interrupt detected, terminating workers!")
pool.terminate()
sys.exit(1)
except Exception as e:
bot.error(e)
return finished