def thread_init():
if len(THREAD_CONFIG):
chalk.green('current time: %r' % utc_now())
chalk.green(
'Found %d thread(s) specified by THREAD_CONFIG' % len(THREAD_CONFIG)
)
for index, thread_group_list in enumerate(THREAD_CONFIG):
runner = Runner(index, thread_group_list)
tiempo_loop.start()
def run_task(job_string, runner):
try:
task = Job.rehydrate(job_string)
try:
runner.current_job = task.current_job
task.run()
except Exception, e:
# print traceback.format_exc()
# task.finish(traceback.format_exc())
raise #####
hxdispatcher.send('all_tasks', {'runner': runner.number,
'time': utc_now().isoformat(),
'message': task.key,
'code_word': task.current_job.code_word})
thread.running_task = None
def finished(self, timestamp=None):
if DEBUG:
pass
# return
self.timestamp = timestamp
if not timestamp:
self.timestamp = utc_now()
task_key = '%s:%s' % (self.task.key, self.task.uid)
expire_time = int(((self.timestamp + relativedelta(
days=RESULT_LIFESPAN)) - self.timestamp).total_seconds())
pipe = REDIS.pipeline()
pipe.zadd(RECENT_KEY, self.start_time.strftime('%s'), task_key)
pipe.set(self.task.uid, self.format_output())
pipe.expire(self.task.uid, expire_time)
pipe.execute()
def run():
this_loop_runtime = utc_now()
# This loop basically does two things:
for runner in all_runners():
# 1) Let the runners pick up any queued tasks.
result = runner.run()
if not result in (BUSY, IDLE):
# If the runner is neither busy nor idle, it will have returned a Deferred.
result.addCallback(runner.finish_job)
# 2) Queue up new tasks.
for task_string, task in TIEMPO_REGISTRY.items():
### REPLACE with task.next_expiration_dt()
if hasattr(task, 'force_interval'):
expire_key = this_loop_runtime + datetime.timedelta(
seconds=task.force_interval
)
else:
expire_key = task_time_keys().get(task.get_schedule())
#########
if expire_key:
stop_key_has_expired = REDIS.setnx(task.stop_key, 0)
if stop_key_has_expired:
REDIS.expire(
task.stop_key,
int(float((expire_key - this_loop_runtime).total_seconds())) - 1
)
# OK, we're ready to queue up a new job for this task!
task.spawn_job()
def __init__(self, *args, **kwargs):
self.task = kwargs.pop('task')
self.start_time = utc_now()
super(CaptureStdOut, self).__init__(*args, **kwargs)