def output(self, run_settings):
"""
Assume that no nodes have finished yet and indicate to future corestages
"""
setvals(run_settings, {
'%s/stages/execute/executed_procs' % RMIT_SCHEMA: str(self.exec_procs),
'%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.schedule_procs),
'%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes)
})
#completed_processes = [x for x in self.exec_procs if x['status'] == 'completed']
completed_processes = [x for x in self.schedule_procs if x['status'] == 'completed']
running_processes = [x for x in self.schedule_procs if x['status'] == 'running']
logger.debug('completed_processes=%d' % len(completed_processes))
setvals(run_settings, {
'%s/stages/run/runs_left' % RMIT_SCHEMA:
len(running_processes),
# len(self.exec_procs) - len(completed_processes),
'%s/stages/run/initial_numbfile' % RMIT_SCHEMA: self.initial_numbfile,
'%s/stages/run/rand_index' % RMIT_SCHEMA: self.rand_index, #fixme remove rand_index
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA: str(self.experiment_id)
})
return run_settings
def output(self, run_settings):
setvals(
run_settings, {
'%s/stages/bootstrap/started' % django_settings.SCHEMA_PREFIX:
self.started,
'%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX:
str(self.bootstrapped_nodes),
'%s/system/id' % django_settings.SCHEMA_PREFIX:
0,
'%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX:
self.created_nodes
})
#todo: move id to hrmc parent subclass parent?? may be not needed
running_created_nodes = [
x for x in self.created_nodes if x[3] == 'running'
]
logger.debug('running created_nodes=%s' % running_created_nodes)
if self.bootstrapped_nodes and len(
self.bootstrapped_nodes) == len(running_created_nodes):
setvals(
run_settings, {
'%s/stages/bootstrap/bootstrap_done' % django_settings.SCHEMA_PREFIX:
1
})
return run_settings
def output(self, run_settings):
"""
Assume that no nodes have finished yet and indicate to future corestages
"""
setvals(run_settings, {
'%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: str(self.exec_procs),
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.schedule_procs),
'%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes)
})
#completed_processes = [x for x in self.exec_procs if x['status'] == 'completed']
completed_processes = [
x for x in self.schedule_procs if x['status'] == 'completed']
running_processes = [
x for x in self.schedule_procs if x['status'] == 'running']
logger.debug('completed_processes=%d' % len(completed_processes))
setvals(run_settings, {
'%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX:
len(running_processes),
# len(self.exec_procs) - len(completed_processes),
'%s/stages/run/initial_numbfile' % django_settings.SCHEMA_PREFIX: self.initial_numbfile,
# fixme remove rand_index
'%s/stages/run/rand_index' % django_settings.SCHEMA_PREFIX: self.rand_index,
'%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX: str(self.experiment_id)
})
return run_settings
def output(self, run_settings):
setval(run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id))
if not self.done_iterating:
# trigger first of iteration corestages
logger.debug("nonconvergence")
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: '[]',
'%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: '[]',
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: '[]',
'%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX: 0,
'%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 0,
'%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 0
})
logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX))
try:
delkey(run_settings, '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
try:
delkey(run_settings, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
# run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]
# del run['error_nodes']
#update_key('converged', False, context)
setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 0)
# run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0
# delete_key('runs_left', context)
# delete_key('error_nodes', context)
# update_key('converged', False, context)
else:
logger.debug("convergence")
# we are done, so trigger next stage outside of converge
#update_key('converged', True, context)
setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 1)
# we are done, so don't trigger iteration stages
#update_key('criterion', self.criterion, context)
setval(run_settings, '%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX, unicode(self.criterion))
# delete_key('error_nodes', context)
#delete_key('transformed', context)
try:
delkey(run_settings, '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
self.id += 1
# update_key('id', self.id, context)
setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX, self.id)
return run_settings
def output(self, run_settings):
logger.debug("transform.output")
setvals(run_settings, {
'%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX: 1,
'%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX: str(self.experiment_id),
})
#print "End of Transformation: \n %s" % self.audit
return run_settings
def output(self, run_settings):
logger.debug("transform.output")
setvals(
run_settings, {
'%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX:
1,
'%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX:
str(self.experiment_id),
})
#print "End of Transformation: \n %s" % self.audit
return run_settings
def output(self, run_settings):
setvals(run_settings, {
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA: self.experiment_id,
'%s/stages/make/runs_left' % RMIT_SCHEMA: str(self.runs_left)
})
# run_settings['http://rmit.edu.au/schemas/stages/make']['runs_left'] \
# = str(self.runs_left)
# run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] \
# = self.experiment_id
# # run_settings['http://rmit.edu.au/schemas/stages/make']['running'] = \
# # self.still_running
return run_settings
def output(self, run_settings):
setvals(
run_settings, {
'%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX:
str(self.scheduled_nodes),
'%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX:
str(self.rescheduled_nodes),
'%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX:
str(self.all_processes),
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX:
str(self.current_processes)
})
if not self.started:
setvals(
run_settings, {
'%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX:
1,
'%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX:
self.procs_2b_rescheduled
})
if not self.procs_2b_rescheduled:
setvals(
run_settings, {
'%s/stages/schedule/total_processes' % django_settings.SCHEMA_PREFIX:
str(self.total_processes),
'%s/stages/schedule/schedule_index' % django_settings.SCHEMA_PREFIX:
self.schedule_index
})
else:
if self.procs_2b_rescheduled:
setval(
run_settings,
'%s/stages/schedule/total_rescheduled_procs' %
django_settings.SCHEMA_PREFIX,
self.total_rescheduled_procs)
if self.total_rescheduled_procs == len(
self.procs_2b_rescheduled):
setvals(
run_settings, {
'%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX:
1,
'%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX:
[],
'%s/stages/schedule/total_rescheduled_procs' % django_settings.SCHEMA_PREFIX:
0,
'%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX:
[],
})
else:
setval(
run_settings, '%s/stages/schedule/total_scheduled_procs' %
django_settings.SCHEMA_PREFIX, self.total_scheduled_procs)
if self.total_scheduled_procs == len(self.current_processes):
setval(
run_settings, '%s/stages/schedule/schedule_completed' %
django_settings.SCHEMA_PREFIX, 1)
return run_settings
def output(self, run_settings):
setvals(run_settings, {
'%s/stages/bootstrap/started' % django_settings.SCHEMA_PREFIX: self.started,
'%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX: str(self.bootstrapped_nodes),
'%s/system/id' % django_settings.SCHEMA_PREFIX: 0,
'%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX: self.created_nodes
})
#todo: move id to hrmc parent subclass parent?? may be not needed
running_created_nodes = [x for x in self.created_nodes if x[3] == 'running']
logger.debug('running created_nodes=%s' % running_created_nodes)
if self.bootstrapped_nodes and len(self.bootstrapped_nodes) == len(running_created_nodes):
setvals(run_settings, {
'%s/stages/bootstrap/bootstrap_done' % django_settings.SCHEMA_PREFIX: 1})
return run_settings
def output(self, run_settings):
setvals(
run_settings, {
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA:
self.experiment_id,
'%s/stages/make/runs_left' % RMIT_SCHEMA: str(self.runs_left)
})
# run_settings['http://rmit.edu.au/schemas/stages/make']['runs_left'] \
# = str(self.runs_left)
# run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] \
# = self.experiment_id
# # run_settings['http://rmit.edu.au/schemas/stages/make']['running'] = \
# # self.still_running
return run_settings
def output(self, run_settings):
# TODO: should only set runnning if program_success is true?
setvals(run_settings, {
'%s/stages/make/program_success' % RMIT_SCHEMA: self.program_success,
'%s/stages/make/running' % RMIT_SCHEMA: 1
})
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/make',
# {})[u'program_success'] = self.program_success
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/make',
# {})[u'running'] = 1
return run_settings
def output(self, run_settings):
logger.debug("transform.output")
# if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/transform'):
# run_settings['http://rmit.edu.au/schemas/stages/transform'] = {}
setvals(run_settings, {
'%s/stages/transform/transformed' % RMIT_SCHEMA: 1,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA: str(self.experiment_id),
})
# run_settings['http://rmit.edu.au/schemas/stages/transform'][u'transformed'] = 1
# run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] = str(self.experiment_id)
#print "End of Transformation: \n %s" % self.audit
return run_settings
def output(self, run_settings):
logger.debug("transform.output")
# if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/transform'):
# run_settings['http://rmit.edu.au/schemas/stages/transform'] = {}
setvals(run_settings, {
'%s/stages/transform/transformed' % RMIT_SCHEMA: 1,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA: str(self.experiment_id),
})
# run_settings['http://rmit.edu.au/schemas/stages/transform'][u'transformed'] = 1
# run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] = str(self.experiment_id)
#print "End of Transformation: \n %s" % self.audit
return run_settings
def output(self, run_settings):
""" produce the resulting datfiles and metadata
"""
logger.debug("CopyDirectory Stage Output")
logger.debug("run_settings=%s" % run_settings)
setvals(run_settings, {
'%s/stages/upload_makefile/done' % RMIT_SCHEMA: 1,
'%s/stages/make/runs_left' % RMIT_SCHEMA: str(1)
})
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/upload_makefile',
# {})[u'done'] = 1
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/make',
# {})[u'runs_left'] = str(1)
return run_settings
def output(self, run_settings):
"""
Output new runs_left value (including zero value)
"""
logger.debug("finished stage output")
executing_procs = [x for x in self.current_processes if x['status'] != 'ready']
#nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes)
nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes)
#if len(self.finished_nodes) == 0 and self.failed_processes == 0:
# nodes_working = 0
logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes))
#logger.debug("self.executed_procs=%s" % self.executed_procs)
logger.debug("self.executed_procs=%s" % executing_procs)
logger.debug("self.finished_nodes=%s" % self.finished_nodes)
#FIXME: nodes_working can be negative
# FIXME: possible race condition?
# if not self._exists(run_settings, '%s/stages/run' % django_settings.SCHEMA_PREFIX):
# run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] = {}
# #run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['runs_left'] = nodes_working
#run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['error_nodes'] = nodes_working
setvals(run_settings, {
'%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX: nodes_working,
'%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX: nodes_working,
})
if not nodes_working:
self.finished_nodes = []
#messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1))
setvals(run_settings, {
'%s/stages/run/finished_nodes' % django_settings.SCHEMA_PREFIX: str(self.finished_nodes),
'%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes),
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.current_processes),
'%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: str(self.executed_procs)
})
if self.failed_nodes:
setval(run_settings, '%s/stages/create/failed_nodes' % django_settings.SCHEMA_PREFIX, self.failed_nodes)
# run_settings.setdefault(
completed_procs = [x for x in self.executed_procs if x['status'] == 'completed']
# messages.info(run_settings, "%s: waiting (%s of %s processes done)"
# % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes)))
if self.procs_2b_rescheduled:
setvals(run_settings, {
'%s/stages/schedule/scheduled_started' % django_settings.SCHEMA_PREFIX: 0,
'%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: self.procs_2b_rescheduled
})
return run_settings
def output(self, run_settings):
setvals(run_settings, {
'%s/stages/destroy/run_finished' % RMIT_SCHEMA: 1
})
setvals(run_settings, {
'%s/stages/create/created_nodes' % RMIT_SCHEMA: self.created_nodes
})
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: self.scheduled_nodes
})
setvals(run_settings, {
'%s/stages/bootstrap/bootstrapped_nodes' % RMIT_SCHEMA: self.bootstrapped_nodes
})
messages.success(run_settings, "%d: finished" % self.id)
return run_settings
def output(self, run_settings):
setvals(run_settings, {
'%s/stages/destroy/run_finished' % django_settings.SCHEMA_PREFIX: 1
})
setvals(run_settings, {
'%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX: self.created_nodes
})
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: self.scheduled_nodes
})
setvals(run_settings, {
'%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX: self.bootstrapped_nodes
})
messages.success(run_settings, "%d: Completed" % self.id)
return run_settings
def output(self, run_settings):
setvals(run_settings,
{'%s/stages/destroy/run_finished' % RMIT_SCHEMA: 1})
setvals(run_settings, {
'%s/stages/create/created_nodes' % RMIT_SCHEMA:
self.created_nodes
})
setvals(
run_settings, {
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA:
self.scheduled_nodes
})
setvals(
run_settings, {
'%s/stages/bootstrap/bootstrapped_nodes' % RMIT_SCHEMA:
self.bootstrapped_nodes
})
messages.success(run_settings, "%d: finished" % self.id)
return run_settings
def output(self, run_settings):
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: str(self.scheduled_nodes),
'%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: str(self.rescheduled_nodes),
'%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes),
'%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes)
})
if not self.started:
setvals(run_settings, {
'%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 1,
'%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled
})
if not self.procs_2b_rescheduled:
setvals(run_settings, {
'%s/stages/schedule/total_processes' % RMIT_SCHEMA: str(self.total_processes),
'%s/stages/schedule/schedule_index' % RMIT_SCHEMA: self.schedule_index
})
else:
if self.procs_2b_rescheduled:
setval(run_settings,
'%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA,
self.total_rescheduled_procs)
if self.total_rescheduled_procs == len(self.procs_2b_rescheduled):
setvals(run_settings, {
'%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 1,
'%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: [],
'%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA: 0,
'%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: [],
})
else:
setval(run_settings,
'%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA,
self.total_scheduled_procs)
if self.total_scheduled_procs == len(self.current_processes):
setval(run_settings,
'%s/stages/schedule/schedule_completed' % RMIT_SCHEMA,
1)
return run_settings
def output(self, run_settings):
# if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/converge'):
# run_settings['http://rmit.edu.au/schemas/stages/converge'] = {}
setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id))
# run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] = str(self.experiment_id)
if not self.done_iterating:
# trigger first of iteration corestages
logger.debug("nonconvergence")
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: '[]',
'%s/stages/execute/executed_procs' % RMIT_SCHEMA: '[]',
'%s/stages/schedule/current_processes' % RMIT_SCHEMA: '[]',
'%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA: 0,
'%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 0,
'%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 0
})
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'scheduled_nodes'] = '[]'
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/execute',
# {})[u'executed_procs'] = '[]'
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'current_processes'] = '[]'
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'total_scheduled_procs'] = 0
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'schedule_completed'] = 0
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'schedule_started'] = 0
logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA))
try:
delkey(run_settings, '%s/stages/run/runs_left' % RMIT_SCHEMA)
except SettingNotFoundException:
pass
# run = run_settings['http://rmit.edu.au/schemas/stages/run']
# del run['runs_left']
try:
delkey(run_settings, '%s/stages/run/error_nodes' % RMIT_SCHEMA)
except SettingNotFoundException:
pass
# run = run_settings['%s/stages/run' % RMIT_SCHEMA]
# del run['error_nodes']
#update_key('converged', False, context)
setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 0)
# run_settings['%s/stages/converge' % RMIT_SCHEMA][u'converged'] = 0
# delete_key('runs_left', context)
# delete_key('error_nodes', context)
# update_key('converged', False, context)
else:
logger.debug("convergence")
# we are done, so trigger next stage outside of converge
#update_key('converged', True, context)
setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 1)
# run_settings['http://rmit.edu.au/schemas/stages/converge'][u'converged'] = 1
# we are done, so don't trigger iteration stages
#update_key('criterion', self.criterion, context)
setval(run_settings, '%s/stages/converge/criterion' % RMIT_SCHEMA, unicode(self.criterion))
# run_settings['http://rmit.edu.au/schemas/stages/converge'][u'criterion'] = unicode(self.criterion)
# delete_key('error_nodes', context)
#delete_key('transformed', context)
try:
delkey(run_settings, '%s/stages/transform/transformed' % RMIT_SCHEMA)
except SettingNotFoundException:
pass
# run = run_settings['http://rmit.edu.au/schemas/stages/transform']
# del run['transformed']
self.id += 1
# update_key('id', self.id, context)
setval(run_settings, '%s/system/id' % RMIT_SCHEMA, self.id)
# run_settings['http://rmit.edu.au/schemas/system'][u'id'] = self.id
return run_settings
def output(self, run_settings):
setval(
run_settings,
'%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX,
str(self.experiment_id))
if not self.done_iterating:
# trigger first of iteration corestages
logger.debug("nonconvergence")
setvals(
run_settings, {
'%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX:
'[]',
'%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX:
'[]',
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX:
'[]',
'%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX:
0,
'%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX:
0,
'%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX:
0
})
logger.debug('scheduled_nodes=%s' % getval(
run_settings, '%s/stages/schedule/scheduled_nodes' %
django_settings.SCHEMA_PREFIX))
try:
delkey(
run_settings,
'%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
try:
delkey(
run_settings, '%s/stages/run/error_nodes' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
# run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]
# del run['error_nodes']
#update_key('converged', False, context)
setval(
run_settings,
'%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX,
0)
# run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0
# delete_key('runs_left', context)
# delete_key('error_nodes', context)
# update_key('converged', False, context)
else:
logger.debug("convergence")
# we are done, so trigger next stage outside of converge
#update_key('converged', True, context)
setval(
run_settings,
'%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX,
1)
# we are done, so don't trigger iteration stages
#update_key('criterion', self.criterion, context)
setval(run_settings,
'%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX,
unicode(self.criterion))
# delete_key('error_nodes', context)
#delete_key('transformed', context)
try:
delkey(
run_settings, '%s/stages/transform/transformed' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
self.id += 1
# update_key('id', self.id, context)
setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX,
self.id)
return run_settings
def output(self, run_settings):
"""
Output new runs_left value (including zero value)
"""
logger.debug("finished stage output")
executing_procs = [x for x in self.current_processes if x['status'] != 'ready']
#nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes)
nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes)
#if len(self.finished_nodes) == 0 and self.failed_processes == 0:
# nodes_working = 0
logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes))
#logger.debug("self.executed_procs=%s" % self.executed_procs)
logger.debug("self.executed_procs=%s" % executing_procs)
logger.debug("self.finished_nodes=%s" % self.finished_nodes)
#FIXME: nodes_working can be negative
# FIXME: possible race condition?
# if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/run'):
# run_settings['http://rmit.edu.au/schemas/stages/run'] = {}
# #run_settings['http://rmit.edu.au/schemas/stages/run']['runs_left'] = nodes_working
#run_settings['http://rmit.edu.au/schemas/stages/run']['error_nodes'] = nodes_working
setvals(run_settings, {
'%s/stages/run/runs_left' % RMIT_SCHEMA: nodes_working,
'%s/stages/run/error_nodes' % RMIT_SCHEMA: nodes_working,
})
if not nodes_working:
self.finished_nodes = []
#messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1))
#run_settings['http://rmit.edu.au/schemas/stages/run']['finished_nodes'] = str(self.finished_nodes)
setvals(run_settings, {
'%s/stages/run/finished_nodes' % RMIT_SCHEMA: str(self.finished_nodes),
'%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes),
'%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes),
'%s/stages/execute/executed_procs' % RMIT_SCHEMA: str(self.executed_procs)
})
if self.failed_nodes:
setval(run_settings, '%s/stages/create/failed_nodes' % RMIT_SCHEMA, self.failed_nodes)
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/create', {})[u'failed_nodes'] = self.failed_nodes
completed_procs = [x for x in self.executed_procs if x['status'] == 'completed']
# messages.info(run_settings, "%s: waiting (%s of %s processes done)"
# % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes)))
if self.procs_2b_rescheduled:
setvals(run_settings, {
'%s/stages/schedule/scheduled_started' % RMIT_SCHEMA: 0,
'%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled
})
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'schedule_started'] = 0
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'procs_2b_rescheduled'] = self.procs_2b_rescheduled
return run_settings
