def schedule_next(task_id, status, errmsg=None):
plan_uid, task_name = task_id.rsplit(":", 1)
dg = graph.get_graph(plan_uid)
dg.node[task_name]["status"] = status
dg.node[task_name]["errmsg"] = errmsg
schedule(plan_uid, dg)
def anchor(ctxt, *args):
# such tasks should be walked when atleast 1/3/exact number of resources
# visited
dg = graph.get_graph('current')
for s in dg.predecessors(ctxt.request.id):
if dg.node[s]['status'] != 'SUCCESS':
raise Exception('One of the tasks erred, cant proceeed')
def dg(uid, start, end):
plan = graph.get_graph(uid)
if start or end:
errors = filters.filter(plan, start=start, end=end)
if errors:
raise click.ClickException('\n'.join(errors))
utils.write_graph(plan)
click.echo('Created {name}.png'.format(name=plan.graph['name']))
def dg(uid, start, end):
plan = graph.get_graph(uid)
if start or end:
errors = filters.filter(plan, start=start, end=end)
if errors:
raise click.ClickException('\n'.join(errors))
utils.write_graph(plan)
click.echo('Created {name}.svg'.format(name=plan.graph['name']))
def schedule_start(plan_uid):
"""On receive finished task should update storage with task result:
- find successors that should be executed
- apply different policies to tasks
"""
dg = graph.get_graph(plan_uid)
schedule(plan_uid, dg)
def soft_stop(self, ctxt, plan_uid):
with Lock(plan_uid, str(get_current_ident()), retries=20, wait=1):
plan = graph.get_graph(plan_uid)
for n in plan:
if plan.node[n]['status'] in (
states.PENDING.name, states.PENDING_RETRY.name):
plan.node[n]['status'] = states.SKIPPED.name
graph.update_graph(plan)
def schedule_next(task_id, status, errmsg=None):
plan_uid, task_name = task_id.rsplit(':', 1)
dg = graph.get_graph(plan_uid)
dg.node[task_name]['status'] = status
dg.node[task_name]['errmsg'] = errmsg
dg.node[task_name]['end_time'] = time.time()
schedule(plan_uid, dg)
def schedule_next(task_id, status, errmsg=None):
plan_uid, task_name = task_id.rsplit(':', 1)
dg = graph.get_graph(plan_uid)
dg.node[task_name]['status'] = status
dg.node[task_name]['errmsg'] = errmsg
dg.node[task_name]['end_time'] = time.time()
schedule(plan_uid, dg)
def schedule_start(plan_uid):
"""On receive finished task should update storage with task result:
- find successors that should be executed
- apply different policies to tasks
"""
dg = graph.get_graph(plan_uid)
schedule(plan_uid, dg)
def filter(uid, start, end):
graph.reset_filtered(uid)
plan = graph.get_graph(uid)
errors = filters.filter(plan, start=start, end=end)
if errors:
raise click.ClickException('\n'.join(errors))
graph.update_graph(plan)
utils.write_graph(plan)
click.echo('Created {name}.png'.format(name=plan.graph['name']))
def filter(uid, start, end):
graph.reset_filtered(uid)
plan = graph.get_graph(uid)
errors = filters.filter(plan, start=start, end=end)
if errors:
raise click.ClickException('\n'.join(errors))
graph.save_graph(uid, plan)
utils.write_graph(plan)
click.echo('Created {name}.png'.format(name=plan.graph['name']))
def test_soft_stop(simple_plan):
uid = simple_plan.graph["uid"]
scheduler = Scheduler(None)
scheduler.soft_stop({}, uid)
plan = graph.get_graph(uid)
for n in plan:
assert n.status == states.SKIPPED.name
def test_soft_stop(simple_plan):
# graph.save_graph(simple_plan)
uid = simple_plan.graph['uid']
scheduler = Scheduler(None)
scheduler.soft_stop({}, uid)
plan = graph.get_graph(uid)
for n in plan:
assert plan.node[n]['status'] == states.SKIPPED.name
def next(self, ctxt, plan_uid):
with Lock(plan_uid, str(get_current_ident()), retries=20, wait=1):
log.debug('Received *next* event for %s', plan_uid)
plan = graph.get_graph(plan_uid)
rst = self._next(plan)
for task_name in rst:
self._do_scheduling(plan, task_name)
graph.update_graph(plan)
log.debug('Scheduled tasks %r', rst)
# process tasks with tasks client
return rst
def soft_stop(self, ctxt, plan_uid):
with Lock(plan_uid,
str(get_current_ident()),
retries=20,
waiter=Waiter(1)):
plan = graph.get_graph(plan_uid)
for n in plan:
if plan.node[n]['status'] in (states.PENDING.name,
states.ERROR_RETRY.name):
plan.node[n]['status'] = states.SKIPPED.name
graph.update_graph(plan)
def soft_stop(self, ctxt, plan_uid):
with Lock(
plan_uid,
str(get_current_ident()),
retries=20,
waiter=Waiter(1)
):
for task in graph.get_graph(plan_uid):
if task.status in (
states.PENDING.name, states.ERROR_RETRY.name):
task.status = states.SKIPPED.name
task.save_lazy()
def update_next(self, ctxt, status, errmsg):
log.debug(
'Received update for TASK %s - %s %s',
ctxt['task_id'], status, errmsg)
plan_uid, task_name = ctxt['task_id'].rsplit(':', 1)
with Lock(plan_uid, str(get_current_ident()), retries=20, wait=1):
plan = graph.get_graph(plan_uid)
self._do_update(plan, task_name, status, errmsg=errmsg)
rst = self._next(plan)
for task_name in rst:
self._do_scheduling(plan, task_name)
graph.update_graph(plan)
log.debug('Scheduled tasks %r', rst)
return rst
def fault_tolerance(ctxt, percent):
task_id = ctxt.request.id
plan_uid, task_name = task_id.rsplit(":", 1)
dg = graph.get_graph(plan_uid)
success = 0.0
predecessors = dg.predecessors(task_name)
lth = len(predecessors)
for s in predecessors:
if dg.node[s]["status"] == "SUCCESS":
success += 1
succes_percent = (success / lth) * 100
if succes_percent < percent:
raise Exception("Cant proceed with, {0} < {1}".format(succes_percent, percent))
def next(self, ctxt, plan_uid):
with Lock(plan_uid,
str(get_current_ident()),
retries=20,
waiter=Waiter(1)):
log.debug('Received *next* event for %s', plan_uid)
plan = graph.get_graph(plan_uid)
if len(plan) == 0:
raise ValueError('Plan {} is empty'.format(plan_uid))
rst = self._next(plan)
for task_name in rst:
self._do_scheduling(plan, task_name)
graph.update_graph(plan)
log.debug('Scheduled tasks %r', rst)
# process tasks with tasks client
return rst
def test_timeout_plan(timeout_plan, scheduler):
worker, client = scheduler
worker._tasks = mock.Mock()
client.next({}, timeout_plan.graph['uid'])
def wait_function(timeout):
for summary in graph.wait_finish(timeout_plan.graph['uid'], timeout):
if summary[states.ERROR.name] == 1:
return summary
time.sleep(0.3)
return summary
waiter = gevent.spawn(wait_function, 2)
waiter.get(block=True, timeout=2)
timeout_plan = graph.get_graph(timeout_plan.graph['uid'])
assert (timeout_plan.node['echo_stuff']['status'] == states.ERROR.name)
def update_next(self, ctxt, status, errmsg):
log.debug('Received update for TASK %s - %s %s', ctxt['task_id'],
status, errmsg)
plan_uid, task_name = ctxt['task_id'].rsplit(':', 1)
with Lock(plan_uid,
str(get_current_ident()),
retries=20,
waiter=Waiter(1)):
plan = graph.get_graph(plan_uid)
self._do_update(plan, task_name, status, errmsg=errmsg)
rst = self._next(plan)
for task_name in rst:
self._do_scheduling(plan, task_name)
graph.update_graph(plan)
log.debug('Scheduled tasks %r', rst)
return rst
def test_timelimit_plan(timelimit_plan, scheduler, tasks):
worker, client = scheduler
client.next({}, timelimit_plan.graph['uid'])
def wait_function(timeout):
try:
for summary in graph.wait_finish(
timelimit_plan.graph['uid'], timeout):
time.sleep(0.5)
except ExecutionTimeout:
return summary
waiter = gevent.spawn(wait_function, 3)
waiter.join(timeout=3)
finished_plan = graph.get_graph(timelimit_plan.graph['uid'])
assert 'ExecutionTimeout' in finished_plan.node['t1']['errmsg']
assert finished_plan.node['t2']['status'] == states.PENDING.name
def fault_tolerance(ctxt, percent):
task_id = ctxt.request.id
plan_uid, task_name = task_id.rsplit(':', 1)
dg = graph.get_graph(plan_uid)
success = 0.0
predecessors = dg.predecessors(task_name)
lth = len(predecessors)
for s in predecessors:
if dg.node[s]['status'] == 'SUCCESS':
success += 1
succes_percent = (success / lth) * 100
if succes_percent < percent:
raise Exception('Cant proceed with, {0} < {1}'.format(
succes_percent, percent))
def test_timelimit_plan(timelimit_plan, scheduler, tasks):
worker, client = scheduler
client.next({}, timelimit_plan.graph['uid'])
def wait_function(timeout):
try:
for summary in graph.wait_finish(timelimit_plan.graph['uid'],
timeout):
time.sleep(0.5)
except ExecutionTimeout:
return summary
waiter = gevent.spawn(wait_function, 3)
waiter.join(timeout=3)
finished_plan = graph.get_graph(timelimit_plan.graph['uid'])
assert 'ExecutionTimeout' in finished_plan.node['t1']['errmsg']
assert finished_plan.node['t2']['status'] == states.PENDING.name
def test_timeout_plan(timeout_plan, scheduler):
worker, client = scheduler
worker._tasks = mock.Mock()
client.next({}, timeout_plan.graph['uid'])
def wait_function(timeout):
for summary in graph.wait_finish(
timeout_plan.graph['uid'], timeout):
if summary[states.ERROR.name] == 1:
return summary
time.sleep(0.3)
return summary
waiter = gevent.spawn(wait_function, 2)
waiter.get(block=True, timeout=2)
timeout_plan = graph.get_graph(timeout_plan.graph['uid'])
echo_task = graph.get_task_by_name(timeout_plan, 'echo_stuff')
assert echo_task.status == states.ERROR.name
def dg(uid):
plan = graph.get_graph(uid)
colors = {
'PENDING': 'cyan',
'ERROR': 'red',
'SUCCESS': 'green',
'INPROGRESS': 'yellow',
'SKIPPED': 'blue'}
for n in plan:
color = colors[plan.node[n]['status']]
plan.node[n]['color'] = color
nx.write_dot(plan, '{name}.dot'.format(name=plan.graph['name']))
subprocess.call(
'tred {name}.dot | dot -Tpng -o {name}.png'.format(name=plan.graph['name']),
shell=True)
click.echo('Created {name}.png'.format(name=plan.graph['name']))
def test_timelimit_plan(timelimit_plan, scheduler, tasks):
worker, client = scheduler
client.next({}, timelimit_plan.graph['uid'])
def wait_function(timeout):
try:
for summary in graph.wait_finish(
timelimit_plan.graph['uid'], timeout):
time.sleep(0.5)
except ExecutionTimeout:
return summary
waiter = gevent.spawn(wait_function, 3)
waiter.join(timeout=3)
finished_plan = graph.get_graph(timelimit_plan.graph['uid'])
t1 = graph.get_task_by_name(finished_plan, 't1')
t2 = graph.get_task_by_name(finished_plan, 't2')
assert 'ExecutionTimeout' in t1.errmsg
assert t2.status == states.PENDING.name
def next(self, ctxt, plan_uid):
with Lock(
plan_uid,
str(get_current_ident()),
retries=20,
waiter=Waiter(1)
):
log.debug('Received *next* event for %s', plan_uid)
plan = graph.get_graph(plan_uid)
if len(plan) == 0:
raise ValueError('Plan {} is empty'.format(plan_uid))
rst = self._next(plan)
for task_name in rst:
self._do_scheduling(plan, task_name)
graph.update_graph(plan)
log.debug('Scheduled tasks %r', rst)
# process tasks with tasks client
return rst
def next(self, ctxt, plan_uid):
with Lock(
plan_uid,
str(get_current_ident()),
retries=20,
waiter=Waiter(1)
):
log.debug('Received *next* event for %s', plan_uid)
plan = graph.get_graph(plan_uid)
# FIXME get_graph should raise DBNotFound if graph is not
# created
if len(plan) == 0:
raise ValueError('Plan {} is empty'.format(plan_uid))
tasks_to_schedule = self._next(plan)
for task in tasks_to_schedule:
self._do_scheduling(task)
log.debug('Scheduled tasks %r', tasks_to_schedule)
ModelMeta.save_all_lazy()
return tasks_to_schedule
def update_next(self, ctxt, status, errmsg):
log.debug(
'Received update for TASK %s - %s %s',
ctxt['task_id'], status, errmsg)
plan_uid, task_name = ctxt['task_id'].rsplit('~', 1)
with Lock(
plan_uid,
str(get_current_ident()),
retries=20,
waiter=Waiter(1)
):
plan = graph.get_graph(plan_uid)
task = next(t for t in plan.nodes() if t.name == task_name)
self._do_update(task, status, errmsg=errmsg)
tasks_to_schedule = self._next(plan)
for task in tasks_to_schedule:
self._do_scheduling(task)
log.debug('Scheduled tasks %r', tasks_to_schedule)
ModelMeta.save_all_lazy()
return tasks_to_schedule
def soft_stop(plan_uid):
dg = graph.get_graph(plan_uid)
for n in dg:
if dg.node[n]["status"] == "PENDING":
dg.node[n]["status"] = "SKIPPED"
graph.save_graph(plan_uid, dg)
def anchor(ctxt, *args):
# such tasks should be walked when atleast 1/3/exact number of resources visited
dg = graph.get_graph("current")
for s in dg.predecessors(ctxt.request.id):
if dg.node[s]["status"] != "SUCCESS":
raise Exception("One of the tasks erred, cant proceeed")
def soft_stop(plan_uid):
dg = graph.get_graph(plan_uid)
for n in dg:
if dg.node[n]['status'] == 'PENDING':
dg.node[n]['status'] = 'SKIPPED'
graph.save_graph(dg)
def soft_stop(plan_uid):
dg = graph.get_graph(plan_uid)
for n in dg:
if dg.node[n]['status'] == 'PENDING':
dg.node[n]['status'] = 'SKIPPED'
graph.save_graph(dg)
