def output(self, run_settings): """ Assume that no nodes have finished yet and indicate to future corestages """ setvals(run_settings, { '%s/stages/execute/executed_procs' % RMIT_SCHEMA: str(self.exec_procs), '%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.schedule_procs), '%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes) }) #completed_processes = [x for x in self.exec_procs if x['status'] == 'completed'] completed_processes = [x for x in self.schedule_procs if x['status'] == 'completed'] running_processes = [x for x in self.schedule_procs if x['status'] == 'running'] logger.debug('completed_processes=%d' % len(completed_processes)) setvals(run_settings, { '%s/stages/run/runs_left' % RMIT_SCHEMA: len(running_processes), # len(self.exec_procs) - len(completed_processes), '%s/stages/run/initial_numbfile' % RMIT_SCHEMA: self.initial_numbfile, '%s/stages/run/rand_index' % RMIT_SCHEMA: self.rand_index, #fixme remove rand_index '%s/input/mytardis/experiment_id' % RMIT_SCHEMA: str(self.experiment_id) }) return run_settings
def output(self, run_settings): setvals( run_settings, { '%s/stages/bootstrap/started' % django_settings.SCHEMA_PREFIX: self.started, '%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX: str(self.bootstrapped_nodes), '%s/system/id' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX: self.created_nodes }) #todo: move id to hrmc parent subclass parent?? may be not needed running_created_nodes = [ x for x in self.created_nodes if x[3] == 'running' ] logger.debug('running created_nodes=%s' % running_created_nodes) if self.bootstrapped_nodes and len( self.bootstrapped_nodes) == len(running_created_nodes): setvals( run_settings, { '%s/stages/bootstrap/bootstrap_done' % django_settings.SCHEMA_PREFIX: 1 }) return run_settings
def output(self, run_settings): """ Assume that no nodes have finished yet and indicate to future corestages """ setvals(run_settings, { '%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: str(self.exec_procs), '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.schedule_procs), '%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes) }) #completed_processes = [x for x in self.exec_procs if x['status'] == 'completed'] completed_processes = [ x for x in self.schedule_procs if x['status'] == 'completed'] running_processes = [ x for x in self.schedule_procs if x['status'] == 'running'] logger.debug('completed_processes=%d' % len(completed_processes)) setvals(run_settings, { '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX: len(running_processes), # len(self.exec_procs) - len(completed_processes), '%s/stages/run/initial_numbfile' % django_settings.SCHEMA_PREFIX: self.initial_numbfile, # fixme remove rand_index '%s/stages/run/rand_index' % django_settings.SCHEMA_PREFIX: self.rand_index, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX: str(self.experiment_id) }) return run_settings
def output(self, run_settings): setval(run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id)) if not self.done_iterating: # trigger first of iteration corestages logger.debug("nonconvergence") setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 0 }) logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX)) try: delkey(run_settings, '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass try: delkey(run_settings, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass # run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] # del run['error_nodes'] #update_key('converged', False, context) setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 0) # run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0 # delete_key('runs_left', context) # delete_key('error_nodes', context) # update_key('converged', False, context) else: logger.debug("convergence") # we are done, so trigger next stage outside of converge #update_key('converged', True, context) setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 1) # we are done, so don't trigger iteration stages #update_key('criterion', self.criterion, context) setval(run_settings, '%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX, unicode(self.criterion)) # delete_key('error_nodes', context) #delete_key('transformed', context) try: delkey(run_settings, '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass self.id += 1 # update_key('id', self.id, context) setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX, self.id) return run_settings
def output(self, run_settings): logger.debug("transform.output") setvals(run_settings, { '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX: 1, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX: str(self.experiment_id), }) #print "End of Transformation: \n %s" % self.audit return run_settings
def output(self, run_settings): logger.debug("transform.output") setvals( run_settings, { '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX: 1, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX: str(self.experiment_id), }) #print "End of Transformation: \n %s" % self.audit return run_settings
def output(self, run_settings): setvals(run_settings, { '%s/input/mytardis/experiment_id' % RMIT_SCHEMA: self.experiment_id, '%s/stages/make/runs_left' % RMIT_SCHEMA: str(self.runs_left) }) # run_settings['http://rmit.edu.au/schemas/stages/make']['runs_left'] \ # = str(self.runs_left) # run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] \ # = self.experiment_id # # run_settings['http://rmit.edu.au/schemas/stages/make']['running'] = \ # # self.still_running return run_settings
def output(self, run_settings): setvals( run_settings, { '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: str(self.scheduled_nodes), '%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX: str(self.rescheduled_nodes), '%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes), '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.current_processes) }) if not self.started: setvals( run_settings, { '%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 1, '%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: self.procs_2b_rescheduled }) if not self.procs_2b_rescheduled: setvals( run_settings, { '%s/stages/schedule/total_processes' % django_settings.SCHEMA_PREFIX: str(self.total_processes), '%s/stages/schedule/schedule_index' % django_settings.SCHEMA_PREFIX: self.schedule_index }) else: if self.procs_2b_rescheduled: setval( run_settings, '%s/stages/schedule/total_rescheduled_procs' % django_settings.SCHEMA_PREFIX, self.total_rescheduled_procs) if self.total_rescheduled_procs == len( self.procs_2b_rescheduled): setvals( run_settings, { '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 1, '%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: [], '%s/stages/schedule/total_rescheduled_procs' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX: [], }) else: setval( run_settings, '%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX, self.total_scheduled_procs) if self.total_scheduled_procs == len(self.current_processes): setval( run_settings, '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX, 1) return run_settings
def output(self, run_settings): setvals(run_settings, { '%s/stages/bootstrap/started' % django_settings.SCHEMA_PREFIX: self.started, '%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX: str(self.bootstrapped_nodes), '%s/system/id' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX: self.created_nodes }) #todo: move id to hrmc parent subclass parent?? may be not needed running_created_nodes = [x for x in self.created_nodes if x[3] == 'running'] logger.debug('running created_nodes=%s' % running_created_nodes) if self.bootstrapped_nodes and len(self.bootstrapped_nodes) == len(running_created_nodes): setvals(run_settings, { '%s/stages/bootstrap/bootstrap_done' % django_settings.SCHEMA_PREFIX: 1}) return run_settings
def output(self, run_settings): setvals( run_settings, { '%s/input/mytardis/experiment_id' % RMIT_SCHEMA: self.experiment_id, '%s/stages/make/runs_left' % RMIT_SCHEMA: str(self.runs_left) }) # run_settings['http://rmit.edu.au/schemas/stages/make']['runs_left'] \ # = str(self.runs_left) # run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] \ # = self.experiment_id # # run_settings['http://rmit.edu.au/schemas/stages/make']['running'] = \ # # self.still_running return run_settings
def output(self, run_settings): # TODO: should only set runnning if program_success is true? setvals(run_settings, { '%s/stages/make/program_success' % RMIT_SCHEMA: self.program_success, '%s/stages/make/running' % RMIT_SCHEMA: 1 }) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/make', # {})[u'program_success'] = self.program_success # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/make', # {})[u'running'] = 1 return run_settings
def output(self, run_settings): logger.debug("transform.output") # if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/transform'): # run_settings['http://rmit.edu.au/schemas/stages/transform'] = {} setvals(run_settings, { '%s/stages/transform/transformed' % RMIT_SCHEMA: 1, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA: str(self.experiment_id), }) # run_settings['http://rmit.edu.au/schemas/stages/transform'][u'transformed'] = 1 # run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] = str(self.experiment_id) #print "End of Transformation: \n %s" % self.audit return run_settings
def output(self, run_settings): """ produce the resulting datfiles and metadata """ logger.debug("CopyDirectory Stage Output") logger.debug("run_settings=%s" % run_settings) setvals(run_settings, { '%s/stages/upload_makefile/done' % RMIT_SCHEMA: 1, '%s/stages/make/runs_left' % RMIT_SCHEMA: str(1) }) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/upload_makefile', # {})[u'done'] = 1 # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/make', # {})[u'runs_left'] = str(1) return run_settings
def output(self, run_settings): """ Output new runs_left value (including zero value) """ logger.debug("finished stage output") executing_procs = [x for x in self.current_processes if x['status'] != 'ready'] #nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes) nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes) #if len(self.finished_nodes) == 0 and self.failed_processes == 0: # nodes_working = 0 logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes)) #logger.debug("self.executed_procs=%s" % self.executed_procs) logger.debug("self.executed_procs=%s" % executing_procs) logger.debug("self.finished_nodes=%s" % self.finished_nodes) #FIXME: nodes_working can be negative # FIXME: possible race condition? # if not self._exists(run_settings, '%s/stages/run' % django_settings.SCHEMA_PREFIX): # run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] = {} # #run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['runs_left'] = nodes_working #run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['error_nodes'] = nodes_working setvals(run_settings, { '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX: nodes_working, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX: nodes_working, }) if not nodes_working: self.finished_nodes = [] #messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1)) setvals(run_settings, { '%s/stages/run/finished_nodes' % django_settings.SCHEMA_PREFIX: str(self.finished_nodes), '%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes), '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.current_processes), '%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: str(self.executed_procs) }) if self.failed_nodes: setval(run_settings, '%s/stages/create/failed_nodes' % django_settings.SCHEMA_PREFIX, self.failed_nodes) # run_settings.setdefault( completed_procs = [x for x in self.executed_procs if x['status'] == 'completed'] # messages.info(run_settings, "%s: waiting (%s of %s processes done)" # % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes))) if self.procs_2b_rescheduled: setvals(run_settings, { '%s/stages/schedule/scheduled_started' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: self.procs_2b_rescheduled }) return run_settings
def output(self, run_settings): setvals(run_settings, { '%s/stages/destroy/run_finished' % RMIT_SCHEMA: 1 }) setvals(run_settings, { '%s/stages/create/created_nodes' % RMIT_SCHEMA: self.created_nodes }) setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: self.scheduled_nodes }) setvals(run_settings, { '%s/stages/bootstrap/bootstrapped_nodes' % RMIT_SCHEMA: self.bootstrapped_nodes }) messages.success(run_settings, "%d: finished" % self.id) return run_settings
def output(self, run_settings): setvals(run_settings, { '%s/stages/destroy/run_finished' % django_settings.SCHEMA_PREFIX: 1 }) setvals(run_settings, { '%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX: self.created_nodes }) setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: self.scheduled_nodes }) setvals(run_settings, { '%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX: self.bootstrapped_nodes }) messages.success(run_settings, "%d: Completed" % self.id) return run_settings
def output(self, run_settings): setvals(run_settings, {'%s/stages/destroy/run_finished' % RMIT_SCHEMA: 1}) setvals(run_settings, { '%s/stages/create/created_nodes' % RMIT_SCHEMA: self.created_nodes }) setvals( run_settings, { '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: self.scheduled_nodes }) setvals( run_settings, { '%s/stages/bootstrap/bootstrapped_nodes' % RMIT_SCHEMA: self.bootstrapped_nodes }) messages.success(run_settings, "%d: finished" % self.id) return run_settings
def output(self, run_settings): setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: str(self.scheduled_nodes), '%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: str(self.rescheduled_nodes), '%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes), '%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes) }) if not self.started: setvals(run_settings, { '%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 1, '%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled }) if not self.procs_2b_rescheduled: setvals(run_settings, { '%s/stages/schedule/total_processes' % RMIT_SCHEMA: str(self.total_processes), '%s/stages/schedule/schedule_index' % RMIT_SCHEMA: self.schedule_index }) else: if self.procs_2b_rescheduled: setval(run_settings, '%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA, self.total_rescheduled_procs) if self.total_rescheduled_procs == len(self.procs_2b_rescheduled): setvals(run_settings, { '%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 1, '%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: [], '%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA: 0, '%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: [], }) else: setval(run_settings, '%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA, self.total_scheduled_procs) if self.total_scheduled_procs == len(self.current_processes): setval(run_settings, '%s/stages/schedule/schedule_completed' % RMIT_SCHEMA, 1) return run_settings
def output(self, run_settings): # if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/converge'): # run_settings['http://rmit.edu.au/schemas/stages/converge'] = {} setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id)) # run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] = str(self.experiment_id) if not self.done_iterating: # trigger first of iteration corestages logger.debug("nonconvergence") setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: '[]', '%s/stages/execute/executed_procs' % RMIT_SCHEMA: '[]', '%s/stages/schedule/current_processes' % RMIT_SCHEMA: '[]', '%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA: 0, '%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 0, '%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 0 }) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'scheduled_nodes'] = '[]' # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/execute', # {})[u'executed_procs'] = '[]' # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'current_processes'] = '[]' # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'total_scheduled_procs'] = 0 # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'schedule_completed'] = 0 # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'schedule_started'] = 0 logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA)) try: delkey(run_settings, '%s/stages/run/runs_left' % RMIT_SCHEMA) except SettingNotFoundException: pass # run = run_settings['http://rmit.edu.au/schemas/stages/run'] # del run['runs_left'] try: delkey(run_settings, '%s/stages/run/error_nodes' % RMIT_SCHEMA) except SettingNotFoundException: pass # run = run_settings['%s/stages/run' % RMIT_SCHEMA] # del run['error_nodes'] #update_key('converged', False, context) setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 0) # run_settings['%s/stages/converge' % RMIT_SCHEMA][u'converged'] = 0 # delete_key('runs_left', context) # delete_key('error_nodes', context) # update_key('converged', False, context) else: logger.debug("convergence") # we are done, so trigger next stage outside of converge #update_key('converged', True, context) setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 1) # run_settings['http://rmit.edu.au/schemas/stages/converge'][u'converged'] = 1 # we are done, so don't trigger iteration stages #update_key('criterion', self.criterion, context) setval(run_settings, '%s/stages/converge/criterion' % RMIT_SCHEMA, unicode(self.criterion)) # run_settings['http://rmit.edu.au/schemas/stages/converge'][u'criterion'] = unicode(self.criterion) # delete_key('error_nodes', context) #delete_key('transformed', context) try: delkey(run_settings, '%s/stages/transform/transformed' % RMIT_SCHEMA) except SettingNotFoundException: pass # run = run_settings['http://rmit.edu.au/schemas/stages/transform'] # del run['transformed'] self.id += 1 # update_key('id', self.id, context) setval(run_settings, '%s/system/id' % RMIT_SCHEMA, self.id) # run_settings['http://rmit.edu.au/schemas/system'][u'id'] = self.id return run_settings
def output(self, run_settings): setval( run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id)) if not self.done_iterating: # trigger first of iteration corestages logger.debug("nonconvergence") setvals( run_settings, { '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 0 }) logger.debug('scheduled_nodes=%s' % getval( run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX)) try: delkey( run_settings, '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass try: delkey( run_settings, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass # run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] # del run['error_nodes'] #update_key('converged', False, context) setval( run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 0) # run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0 # delete_key('runs_left', context) # delete_key('error_nodes', context) # update_key('converged', False, context) else: logger.debug("convergence") # we are done, so trigger next stage outside of converge #update_key('converged', True, context) setval( run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 1) # we are done, so don't trigger iteration stages #update_key('criterion', self.criterion, context) setval(run_settings, '%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX, unicode(self.criterion)) # delete_key('error_nodes', context) #delete_key('transformed', context) try: delkey( run_settings, '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass self.id += 1 # update_key('id', self.id, context) setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX, self.id) return run_settings
def output(self, run_settings): """ Output new runs_left value (including zero value) """ logger.debug("finished stage output") executing_procs = [x for x in self.current_processes if x['status'] != 'ready'] #nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes) nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes) #if len(self.finished_nodes) == 0 and self.failed_processes == 0: # nodes_working = 0 logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes)) #logger.debug("self.executed_procs=%s" % self.executed_procs) logger.debug("self.executed_procs=%s" % executing_procs) logger.debug("self.finished_nodes=%s" % self.finished_nodes) #FIXME: nodes_working can be negative # FIXME: possible race condition? # if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/run'): # run_settings['http://rmit.edu.au/schemas/stages/run'] = {} # #run_settings['http://rmit.edu.au/schemas/stages/run']['runs_left'] = nodes_working #run_settings['http://rmit.edu.au/schemas/stages/run']['error_nodes'] = nodes_working setvals(run_settings, { '%s/stages/run/runs_left' % RMIT_SCHEMA: nodes_working, '%s/stages/run/error_nodes' % RMIT_SCHEMA: nodes_working, }) if not nodes_working: self.finished_nodes = [] #messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1)) #run_settings['http://rmit.edu.au/schemas/stages/run']['finished_nodes'] = str(self.finished_nodes) setvals(run_settings, { '%s/stages/run/finished_nodes' % RMIT_SCHEMA: str(self.finished_nodes), '%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes), '%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes), '%s/stages/execute/executed_procs' % RMIT_SCHEMA: str(self.executed_procs) }) if self.failed_nodes: setval(run_settings, '%s/stages/create/failed_nodes' % RMIT_SCHEMA, self.failed_nodes) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/create', {})[u'failed_nodes'] = self.failed_nodes completed_procs = [x for x in self.executed_procs if x['status'] == 'completed'] # messages.info(run_settings, "%s: waiting (%s of %s processes done)" # % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes))) if self.procs_2b_rescheduled: setvals(run_settings, { '%s/stages/schedule/scheduled_started' % RMIT_SCHEMA: 0, '%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled }) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'schedule_started'] = 0 # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'procs_2b_rescheduled'] = self.procs_2b_rescheduled return run_settings