def _check_expired_tasks(self):
time_now = int(now())
if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec
return
self._ttl_check_last_run = time_now
TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000
for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items():
scheduled_task = self.scheduler_tasks.get(scheduled_task_name)
if (scheduled_task_history.get('next_run')
and scheduled_task_history.get('scheduled_task_id')
and (time_now - scheduled_task_history.get('next_run')) > (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT)*1000):
task_id = scheduled_task_history.get('scheduled_task_id')
log.info('Fix broken task id={}, name={}'.format(task_id, scheduled_task_name))
# Get task object from redis key
key = SETTINGS.TASK_STORAGE_KEY.format(scheduled_task_history.get('scheduled_task_id')).encode('utf-8')
task_obj = yield from self.connection.get(key)
# Deserialize task object
try:
if not task_obj:
raise TypeError()
task = Task.deserialize(task_obj)
if task.status != Task.SUCCESSFUL:
# Update task object status
task = task._replace(status=Task.FAILED)
# Set new status to redis
yield from self.connection.set(key, task.serialize(), expire=SETTINGS.TASK_STORAGE_EXPIRE)
except TypeError as ex:
task = None
log.error("Wrong task id={}".format(scheduled_task_history.get('scheduled_task_id')), exc_info=True)
yield from self.connection.delete([key])
# Publish message about finish (FAILED)
if task:
yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), task.status.encode('utf-8'))
else:
yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), Task.FAILED.encode('utf-8'))
# Update scheduler information
# Store next_run in scheduled
try:
task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8'))
task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj)
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0)
scheduled_task_history['next_run'] = 0
task_scheduler = task_scheduler._replace(last_run=task_scheduler.next_run, next_run=0, scheduled_task_id=None)
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
task_scheduler = task_scheduler._replace(next_run=0, scheduled_task_id=None)
yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize())
except:
# We lost SCHEDULER_HISTORY_HASH in db
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0)
scheduled_task_history['next_run'] = 0
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
def create_task(self, name, task_type, run_at, ttl, kwargs, store_to):
task_id = hashlib.md5(
ujson.dumps([
codecs.encode(os.urandom(16), 'hex_codec'), name, task_type,
run_at, ttl, kwargs, store_to
]).encode('utf-8')).hexdigest()
if store_to == Task.STORE_TO_KEY:
store_to = 'STORE_TO_KEY:' + SETTINGS.TASK_RESULTS_STORAGE_KEY.format(
task_id)
task = Task(id=task_id,
name=name,
type=task_type,
kwargs=kwargs,
run_at=run_at,
ttl=ttl,
status=Task.SCHEDULED,
store_to=store_to)
return task
def _deserialize_task(self, raw_task):
try:
task_obj = yield from self.connection.get(SETTINGS.TASK_STORAGE_KEY.format(raw_task.decode('utf-8')).encode('utf-8'))
if not task_obj:
raise TypeError()
task = Task.deserialize(task_obj)
log.info("Got new task id={}, type={}, status={}".format(task.id, task.type, task.status))
if not task.status == Task.SCHEDULED:
log.error("Wrong status={} for task id={}, type={}; Should be SCHEDULED".format(task.status, task.id, task.type))
return
return task
except TypeError as ex:
log.error("Wrong task_id {}".format(raw_task))
return
except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE, value=raw_task)
log.error("Wrong message in queue", exc_info=True)
return
def handle_task_post():
# TODO: needs to check for existence in request
# TODO: needs to check for uniqueness in name (for this user)
task = Task()
task.name = request.form['name']
task.workflow = request.form['workflow']
task.metadata = request.form['extra_info']
nodes = Node.query(Node.workflow == task.workflow,
Node.parent_node == None).fetch(100)
if len(nodes) < 1:
flash("No valid root nodes found for this workflow." +
" Are you sure this workflow exists?")
return "error"
if len(nodes) > 1:
flash("More than one root node exists for this workflow." +
" Something has gone terribly wrong somewhere...")
return "error"
root_node = nodes[0]
task.active_nodes.append(root_node)
# TODO: figure out what the return value is here and check it
task.put()
return "success"
def _deserialize_task(self, raw_task):
try:
task_obj = yield from self.connection.get(
SETTINGS.TASK_STORAGE_KEY.format(
raw_task.decode('utf-8')).encode('utf-8'))
if not task_obj:
raise TypeError()
task = Task.deserialize(task_obj)
log.info("Got new task id={}, type={}, status={}".format(
task.id, task.type, task.status))
if not task.status == Task.SCHEDULED:
log.error(
"Wrong status={} for task id={}, type={}; Should be SCHEDULED"
.format(task.status, task.id, task.type))
return
return task
except TypeError as ex:
log.error("Wrong task_id {}".format(raw_task))
return
except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE,
value=raw_task)
log.error("Wrong message in queue", exc_info=True)
return
def handle_task_post():
# TODO: needs to check for existence in request
# TODO: needs to check for uniqueness in name (for this user)
task = Task()
task.name = request.form['name']
task.workflow = request.form['workflow']
task.metadata = request.form['extra_info']
nodes = Node.query(Node.workflow == task.workflow,
Node.parent_node == None).fetch(100)
if len(nodes) < 1:
flash("No valid root nodes found for this workflow."
+ " Are you sure this workflow exists?")
return "error"
if len(nodes) > 1:
flash("More than one root node exists for this workflow."
+ " Something has gone terribly wrong somewhere...")
return "error"
root_node = nodes[0]
task.active_nodes.append(root_node)
# TODO: figure out what the return value is here and check it
task.put()
return "success"
def handle_tasks_json():
return json.dumps([task.to_json() for task in Task.query().fetch(100)])
def handle_tasks():
tasks = Task.query().fetch(100)
return render_template("tasks.html", tasks=tasks)
def _check_expired_tasks(self):
time_now = int(now())
if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec
return
self._ttl_check_last_run = time_now
TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000
for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items(
):
scheduled_task = self.scheduler_tasks.get(scheduled_task_name)
if (scheduled_task_history.get('next_run')
and scheduled_task_history.get('scheduled_task_id')
and (time_now - scheduled_task_history.get('next_run')) >
(scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT) *
1000):
task_id = scheduled_task_history.get('scheduled_task_id')
log.info('Fix broken task id={}, name={}'.format(
task_id, scheduled_task_name))
# Get task object from redis key
key = SETTINGS.TASK_STORAGE_KEY.format(
scheduled_task_history.get('scheduled_task_id')).encode(
'utf-8')
task_obj = yield from self.connection.get(key)
# Deserialize task object
try:
if not task_obj:
raise TypeError()
task = Task.deserialize(task_obj)
if task.status != Task.SUCCESSFUL:
# Update task object status
task = task._replace(status=Task.FAILED)
# Set new status to redis
yield from self.connection.set(
key,
task.serialize(),
expire=SETTINGS.TASK_STORAGE_EXPIRE)
except TypeError as ex:
task = None
log.error("Wrong task id={}".format(
scheduled_task_history.get('scheduled_task_id')),
exc_info=True)
yield from self.connection.delete([key])
# Publish message about finish (FAILED)
if task:
yield from self.connection.publish(
SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'),
task.status.encode('utf-8'))
else:
yield from self.connection.publish(
SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'),
Task.FAILED.encode('utf-8'))
# Update scheduler information
# Store next_run in scheduled
try:
task_scheduler_obj = yield from self.connection.hget(
SETTINGS.SCHEDULER_HISTORY_HASH,
scheduled_task_name.encode('utf-8'))
task_scheduler = SchedulerTaskHistory.deserialize(
task_scheduler_obj)
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history[
'last_run'] = scheduled_task_history.get(
'next_run', 0)
scheduled_task_history['next_run'] = 0
task_scheduler = task_scheduler._replace(
last_run=task_scheduler.next_run,
next_run=0,
scheduled_task_id=None)
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
task_scheduler = task_scheduler._replace(
next_run=0, scheduled_task_id=None)
yield from self.connection.hset(
SETTINGS.SCHEDULER_HISTORY_HASH,
task_scheduler.name.encode('utf-8'),
task_scheduler.serialize())
except:
# We lost SCHEDULER_HISTORY_HASH in db
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history[
'last_run'] = scheduled_task_history.get(
'next_run', 0)
scheduled_task_history['next_run'] = 0
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
def handle_tasks_json():
return json.dumps(
[task.to_json() for task in Task.query().fetch(100)])
def handle_tasks():
tasks = Task.query().fetch(100)
return render_template("tasks.html", tasks=tasks)
def testModelTask_to_json(self):
task = Task(name='test',
workflow='test workflow',
active_nodes=[ndb.Key('Node', 1)],
metadata='test metadata')