def output(self, run_settings): setval(run_settings, '%s/stages/configure/configure_done' % RMIT_SCHEMA, 1) setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id)) if self.output_exists(run_settings): if self.output_platform_name: run_settings.setdefault( RMIT_SCHEMA + '/platform/storage/output', {})[u'platform_url'] = self.output_platform_name #if self.output_platform_offset: # run_settings[RMIT_SCHEMA + '/platform/storage/output']['offset'] = self.output_platform_offset #else: run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset'] = self.output_loc_offset if self.input_exists(run_settings): if self.input_platform_name: run_settings.setdefault( RMIT_SCHEMA + '/platform/storage/input', {})[u'platform_url'] = self.input_platform_name if self.input_platform_offset: run_settings[RMIT_SCHEMA + '/platform/storage/input']['offset'] = self.input_platform_offset logger.debug('self.compute_platform_name=%s' % self.compute_platform_name) if self.compute_platform_name: run_settings.setdefault('http://rmit.edu.au/schemas/platform/computation', {})[u'platform_url'] = self.compute_platform_name if self.compute_platform_offset: run_settings.setdefault('http://rmit.edu.au/schemas/platform/computation', {})[u'offset'] = self.compute_platform_offset return run_settings
def output(self, run_settings): """ Output new runs_left value (including zero value) """ logger.debug("finished stage output") executing_procs = [x for x in self.current_processes if x['status'] != 'ready'] #nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes) nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes) #if len(self.finished_nodes) == 0 and self.failed_processes == 0: # nodes_working = 0 logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes)) #logger.debug("self.executed_procs=%s" % self.executed_procs) logger.debug("self.executed_procs=%s" % executing_procs) logger.debug("self.finished_nodes=%s" % self.finished_nodes) #FIXME: nodes_working can be negative # FIXME: possible race condition? # if not self._exists(run_settings, '%s/stages/run' % django_settings.SCHEMA_PREFIX): # run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] = {} # #run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['runs_left'] = nodes_working #run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['error_nodes'] = nodes_working setvals(run_settings, { '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX: nodes_working, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX: nodes_working, }) if not nodes_working: self.finished_nodes = [] #messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1)) setvals(run_settings, { '%s/stages/run/finished_nodes' % django_settings.SCHEMA_PREFIX: str(self.finished_nodes), '%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes), '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.current_processes), '%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: str(self.executed_procs) }) if self.failed_nodes: setval(run_settings, '%s/stages/create/failed_nodes' % django_settings.SCHEMA_PREFIX, self.failed_nodes) # run_settings.setdefault( completed_procs = [x for x in self.executed_procs if x['status'] == 'completed'] # messages.info(run_settings, "%s: waiting (%s of %s processes done)" # % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes))) if self.procs_2b_rescheduled: setvals(run_settings, { '%s/stages/schedule/scheduled_started' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: self.procs_2b_rescheduled }) return run_settings
def output(self, run_settings): setval(run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id)) if not self.done_iterating: # trigger first of iteration corestages logger.debug("nonconvergence") setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 0 }) logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX)) try: delkey(run_settings, '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass try: delkey(run_settings, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass # run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] # del run['error_nodes'] #update_key('converged', False, context) setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 0) # run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0 # delete_key('runs_left', context) # delete_key('error_nodes', context) # update_key('converged', False, context) else: logger.debug("convergence") # we are done, so trigger next stage outside of converge #update_key('converged', True, context) setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 1) # we are done, so don't trigger iteration stages #update_key('criterion', self.criterion, context) setval(run_settings, '%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX, unicode(self.criterion)) # delete_key('error_nodes', context) #delete_key('transformed', context) try: delkey(run_settings, '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass self.id += 1 # update_key('id', self.id, context) setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX, self.id) return run_settings
def output(self, run_settings): """ Inserting a new group if into run settings. """ logger.debug('output') setval(run_settings, "%s/stages/create/created_nodes" % RMIT_SCHEMA, self.created_nodes) setval(run_settings, "%s/stages/create/create_done" % RMIT_SCHEMA, 1) logger.debug("Updated run settings %s" % run_settings) return run_settings
def output(self, run_settings): self.writeout_output(run_settings) self.writeout_input(run_settings) self.writeout_computation(run_settings) setval(run_settings, '%s/stages/configure/configure_done' % django_settings.SCHEMA_PREFIX, 1) setval(run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id)) return run_settings
def output(self, run_settings): logger.debug("sweep output") setval(run_settings, '%s/stages/sweep/sweep_done' % django_settings.SCHEMA_PREFIX, 1) logger.debug('interesting run_settings=%s' % run_settings) with ignored(SettingNotFoundException): if getvals(run_settings, '%s/input/mytardis' % django_settings.SCHEMA_PREFIX): setval( run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id)) if not self.error_detected: messages.success(run_settings, "0: sweep completed") return run_settings
def output(self, run_settings): logger.debug("sweep output") setval(run_settings, '%s/stages/sweep/sweep_done' % RMIT_SCHEMA, 1) logger.debug('interesting run_settings=%s' % run_settings) try: if getvals(run_settings, '%s/input/mytardis' % RMIT_SCHEMA): setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id)) except SettingNotFoundException: pass if not self.error_detected: messages.success(run_settings, "0: completed") return run_settings
def output(self, run_settings): setvals( run_settings, { '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: str(self.scheduled_nodes), '%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX: str(self.rescheduled_nodes), '%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes), '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.current_processes) }) if not self.started: setvals( run_settings, { '%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 1, '%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: self.procs_2b_rescheduled }) if not self.procs_2b_rescheduled: setvals( run_settings, { '%s/stages/schedule/total_processes' % django_settings.SCHEMA_PREFIX: str(self.total_processes), '%s/stages/schedule/schedule_index' % django_settings.SCHEMA_PREFIX: self.schedule_index }) else: if self.procs_2b_rescheduled: setval( run_settings, '%s/stages/schedule/total_rescheduled_procs' % django_settings.SCHEMA_PREFIX, self.total_rescheduled_procs) if self.total_rescheduled_procs == len( self.procs_2b_rescheduled): setvals( run_settings, { '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 1, '%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: [], '%s/stages/schedule/total_rescheduled_procs' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX: [], }) else: setval( run_settings, '%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX, self.total_scheduled_procs) if self.total_scheduled_procs == len(self.current_processes): setval( run_settings, '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX, 1) return run_settings
def output(self, run_settings): setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: str(self.scheduled_nodes), '%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: str(self.rescheduled_nodes), '%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes), '%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes) }) if not self.started: setvals(run_settings, { '%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 1, '%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled }) if not self.procs_2b_rescheduled: setvals(run_settings, { '%s/stages/schedule/total_processes' % RMIT_SCHEMA: str(self.total_processes), '%s/stages/schedule/schedule_index' % RMIT_SCHEMA: self.schedule_index }) else: if self.procs_2b_rescheduled: setval(run_settings, '%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA, self.total_rescheduled_procs) if self.total_rescheduled_procs == len(self.procs_2b_rescheduled): setvals(run_settings, { '%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 1, '%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: [], '%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA: 0, '%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: [], }) else: setval(run_settings, '%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA, self.total_scheduled_procs) if self.total_scheduled_procs == len(self.current_processes): setval(run_settings, '%s/stages/schedule/schedule_completed' % RMIT_SCHEMA, 1) return run_settings
class Sweep(Stage): def __init__(self, user_settings=None): self.numbfile = 0 logger.debug("Sweep stage initialized") def is_triggered(self, run_settings): logger.debug('run_settings=%s' % run_settings) try: configure_done = int(getval(run_settings, '%s/stages/sweep/sweep_done' % RMIT_SCHEMA)) except (ValueError, SettingNotFoundException): return True return not configure_done def _get_sweep_name(self, run_settings): try: sweep_name = getval(run_settings, '%s/directive_profile/sweep_name' % RMIT_SCHEMA) except SettingNotFoundException: sweep_name = 'unknown_sweep' return sweep_name def process(self, run_settings): logger.debug('run_settings=%s' % run_settings) # Need to make copy because we pass on run_settings to sub connector # so any changes we make here to run_settings WILL be inherited def make_local_settings(run_settings): from copy import deepcopy local_settings = deepcopy(getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)) update(local_settings, run_settings, RMIT_SCHEMA + '/system/platform', # RMIT_SCHEMA + '/input/mytardis/experiment_id', # RMIT_SCHEMA + '/system/random_numbers', ) local_settings['bdp_username'] = getval( run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) return local_settings local_settings = make_local_settings(run_settings) logger.debug('local_settings=%s' % local_settings) setval(run_settings, '%s/platform/computation/platform_url' % RMIT_SCHEMA, getval(run_settings, '%s/input/system/compplatform/computation_platform' % RMIT_SCHEMA)) def _parse_output_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return name, offset contextid = int(getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)) logger.debug("contextid=%s" % contextid) sweep_name = self._get_sweep_name(run_settings) logger.debug("sweep_name=%s" % sweep_name) output_loc = self.output_exists(run_settings) location = "" if output_loc: location = getval(run_settings, output_loc) output_storage_name, output_storage_offset = \ _parse_output_location(run_settings, location) setval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA, output_storage_name) setval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA, os.path.join(output_storage_offset, '%s%s' % (sweep_name, contextid))) def _parse_input_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return (name, offset) input_loc = self.input_exists(run_settings) if input_loc: location = getval(run_settings, input_loc) input_storage_name, input_storage_offset = \ _parse_input_location(run_settings, location) setval(run_settings, '%s/platform/storage/input/platform_url' % RMIT_SCHEMA, input_storage_name) # store offsets setval(run_settings, '%s/platform/storage/input/offset' % RMIT_SCHEMA, input_storage_offset) # TODO: replace with scratch space computation platform space self.scratch_platform = '%s%s%s' % ( manage.get_scratch_platform(), sweep_name, contextid) # mytardis if output_loc: try: self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA)) except KeyError: self.experiment_id = 0 except ValueError: self.experiment_id = 0 try: curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA) except SettingNotFoundException: curate_data = False if curate_data: self.experiment_id = self.curate_data(run_settings, location, self.experiment_id) setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id)) # generate all variations map_text = getval(run_settings, '%s/input/sweep/sweep_map' % RMIT_SCHEMA) # map_text = run_settings[RMIT_SCHEMA + '/input/sweep']['sweep_map'] sweep_map = json.loads(map_text) logger.debug("sweep_map=%s" % pformat(sweep_map)) runs = _expand_variations(maps=[sweep_map], values={}) logger.debug("runs=%s" % runs) # Create random numbers if needed # TODO: move iseed out of hrmc into separate generic schema # to use on any sweepable connector and make this function # completely hrmc independent. rands = [] try: self.rand_index = getval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA) logger.debug("rand_index=%s" % self.rand_index) except SettingNotFoundException: pass else: # prep random seeds for each run based off original iseed # FIXME: inefficient for large random file # TODO, FIXME: this is potentially problematic if different # runs end up overlapping in the random numbers they utilise. # solution is to have separate random files per run or partition # big file up. try: num_url = getval(run_settings, "%s/system/random_numbers" % RMIT_SCHEMA) logger.debug('num_url=%s' % num_url) except SettingNotFoundException: pass else: try: local_settings['random_numbers'] = num_url rands = generate_rands(settings=local_settings, start_range=0, end_range=-1, num_required=len(runs), start_index=self.rand_index) logger.debug("rands=%s" % rands) except Exception, e: logger.debug('error') logger.error(e) raise # load initial values map in the input directory which # contains variable to use for all subdirectives starting_map = {} if input_loc: input_storage_settings = self.get_platform_settings( run_settings, 'http://rmit.edu.au/schemas/platform/storage/input') try: input_prefix = '%s://%s@' % (input_storage_settings['scheme'], input_storage_settings['type']) values_url = get_url_with_credentials( input_storage_settings, input_prefix + os.path.join(input_storage_settings['ip_address'], input_storage_offset, "initial", VALUES_MAP_FILE), is_relative_path=False) logger.debug("values_url=%s" % values_url) values_e_url = get_url_with_credentials( local_settings, values_url, is_relative_path=False) logger.debug("values_url=%s" % values_e_url) values_content = get_file(values_e_url) logger.debug("values_content=%s" % values_content) starting_map = dict(json.loads(values_content)) except IOError: logger.warn("no starting values file found") except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass logger.debug("starting_map after initial values=%s" % pformat(starting_map)) # Copy form input values info starting map # FIXME: could have name collisions between form inputs and # starting values. for ns in run_settings: if ns.startswith(RMIT_SCHEMA + "/input"): # for k, v in run_settings[ns].items(): for k, v in getvals(run_settings, ns).items(): starting_map[k] = v logger.debug("starting_map after form=%s" % pformat(starting_map)) # FIXME: we assume we will always have input directory # Get input_url directory input_url = "" if input_loc: input_prefix = '%s://%s@' % (input_storage_settings['scheme'], input_storage_settings['type']) input_url = get_url_with_credentials(input_storage_settings, input_prefix + os.path.join(input_storage_settings['ip_address'], input_storage_offset), is_relative_path=False) logger.debug("input_url=%s" % input_url) current_context = models.Context.objects.get(id=contextid) user = current_context.owner.user.username # For each of the generated runs, copy across initial input # to individual input directories with variation values, # and then schedule subrun of sub directive logger.debug("run_settings=%s" % run_settings) for i, context in enumerate(runs): run_counter = int(context['run_counter']) logger.debug("run_counter=%s" % run_counter) run_inputdir = os.path.join(self.scratch_platform, SUBDIRECTIVE_DIR % {'run_counter': str(run_counter)}, FIRST_ITERATION_DIR,) logger.debug("run_inputdir=%s" % run_inputdir) run_iter_url = get_url_with_credentials(local_settings, run_inputdir, is_relative_path=False) logger.debug("run_iter_url=%s" % run_iter_url) # Duplicate any input_directory into runX duplicates if input_loc: logger.debug("context=%s" % context) logger.debug("systemsettings=%s" % pformat(getvals(run_settings, RMIT_SCHEMA + '/input/system'))) copy_directories(input_url, run_iter_url) # Need to load up existing values, because original input_dir could # have contained values for the whole run # This code is deprecated in favour of single values file. self.error_detected = False try: template_name = getval(run_settings, '%s/stages/sweep/template_name' % RMIT_SCHEMA) except SettingNotFoundException: pass else: logger.debug("template_name=%s" % template_name) v_map = {} try: values_url = get_url_with_credentials( local_settings, os.path.join(run_inputdir, "initial", VALUES_MAP_TEMPLATE_FILE % {'template_name': template_name}), is_relative_path=False) logger.debug("values_url=%s" % values_url) values_content = get_file(values_url) logger.debug("values_content=%s" % values_content) v_map = dict(json.loads(values_content), indent=4) except IOError: logger.warn("no values file found") except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass v_map.update(starting_map) v_map.update(context) logger.debug("new v_map=%s" % v_map) put_file(values_url, json.dumps(v_map, indent=4)) v_map = {} try: values_url = get_url_with_credentials( local_settings, os.path.join(run_inputdir, "initial", VALUES_MAP_FILE), is_relative_path=False) logger.debug("values_url=%s" % values_url) values_content = get_file(values_url) logger.debug("values_content=%s" % values_content) v_map = dict(json.loads(values_content), ) except IOError: logger.warn("no values file found") except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass v_map.update(starting_map) v_map.update(context) logger.debug("new v_map=%s" % v_map) put_file(values_url, json.dumps(v_map, indent=4)) # Set random numbers for subdirective logger.debug("run_settings=%s" % pformat(run_settings)) if rands: setval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA, rands[i]) if input_loc: # Set revised input_location for subdirective setval(run_settings, input_loc, "%s/%s/%s" % (self.scratch_platform, SUBDIRECTIVE_DIR % {'run_counter': str(run_counter)}, FIRST_ITERATION_DIR)) # Redirect input run_input_storage_name, run_input_storage_offset = \ _parse_input_location(run_settings, "local/sweep%s/run%s/input_0" % (contextid, run_counter)) # setval(run_settings, # '%s/platform/storage/input/platform_url' % RMIT_SCHEMA, # run_input_storage_name) # setval(run_settings, # '%s/platform/storage/input/offset' % RMIT_SCHEMA, # run_input_storage_offset) logger.debug("run_settings=%s" % pformat(run_settings)) try: _submit_subdirective("nectar", run_settings, user, current_context) except Exception, e: logger.error(e) raise e
def output(self, run_settings): # if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/converge'): # run_settings['http://rmit.edu.au/schemas/stages/converge'] = {} setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id)) # run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] = str(self.experiment_id) if not self.done_iterating: # trigger first of iteration corestages logger.debug("nonconvergence") setvals(run_settings, { '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: '[]', '%s/stages/execute/executed_procs' % RMIT_SCHEMA: '[]', '%s/stages/schedule/current_processes' % RMIT_SCHEMA: '[]', '%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA: 0, '%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 0, '%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 0 }) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'scheduled_nodes'] = '[]' # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/execute', # {})[u'executed_procs'] = '[]' # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'current_processes'] = '[]' # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'total_scheduled_procs'] = 0 # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'schedule_completed'] = 0 # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'schedule_started'] = 0 logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA)) try: delkey(run_settings, '%s/stages/run/runs_left' % RMIT_SCHEMA) except SettingNotFoundException: pass # run = run_settings['http://rmit.edu.au/schemas/stages/run'] # del run['runs_left'] try: delkey(run_settings, '%s/stages/run/error_nodes' % RMIT_SCHEMA) except SettingNotFoundException: pass # run = run_settings['%s/stages/run' % RMIT_SCHEMA] # del run['error_nodes'] #update_key('converged', False, context) setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 0) # run_settings['%s/stages/converge' % RMIT_SCHEMA][u'converged'] = 0 # delete_key('runs_left', context) # delete_key('error_nodes', context) # update_key('converged', False, context) else: logger.debug("convergence") # we are done, so trigger next stage outside of converge #update_key('converged', True, context) setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 1) # run_settings['http://rmit.edu.au/schemas/stages/converge'][u'converged'] = 1 # we are done, so don't trigger iteration stages #update_key('criterion', self.criterion, context) setval(run_settings, '%s/stages/converge/criterion' % RMIT_SCHEMA, unicode(self.criterion)) # run_settings['http://rmit.edu.au/schemas/stages/converge'][u'criterion'] = unicode(self.criterion) # delete_key('error_nodes', context) #delete_key('transformed', context) try: delkey(run_settings, '%s/stages/transform/transformed' % RMIT_SCHEMA) except SettingNotFoundException: pass # run = run_settings['http://rmit.edu.au/schemas/stages/transform'] # del run['transformed'] self.id += 1 # update_key('id', self.id, context) setval(run_settings, '%s/system/id' % RMIT_SCHEMA, self.id) # run_settings['http://rmit.edu.au/schemas/system'][u'id'] = self.id return run_settings
def process(self, run_settings): logger.debug('run_settings=%s' % run_settings) messages.info(run_settings, "0: sweep started") # Need to make copy because we pass on run_settings to sub connector # so any changes we make here to run_settings WILL be inherited def make_local_settings(run_settings): from copy import deepcopy local_settings = deepcopy( getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)) update( local_settings, run_settings, django_settings.SCHEMA_PREFIX + '/system/platform', # django_settings.SCHEMA_PREFIX + '/input/mytardis/experiment_id', # django_settings.SCHEMA_PREFIX + '/system/random_numbers', ) local_settings['bdp_username'] = getval( run_settings, '%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX) return local_settings local_settings = make_local_settings(run_settings) logger.debug('local_settings=%s' % local_settings) compplatform = [ k for k, v in run_settings.iteritems() if k.startswith('%s/input/system/compplatform' % django_settings.SCHEMA_PREFIX) ] setval( run_settings, '%s/platform/computation/platform_url' % django_settings.SCHEMA_PREFIX, getval(run_settings, '%s/computation_platform' % compplatform[0])) def _parse_output_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return name, offset contextid = int( getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX)) logger.debug("contextid=%s" % contextid) sweep_name = self._get_sweep_name(run_settings) logger.debug("sweep_name=%s" % sweep_name) output_loc = self.output_exists(run_settings) location = "" if output_loc: location = getval(run_settings, output_loc) output_storage_name, output_storage_offset = \ _parse_output_location(run_settings, location) setval( run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX, output_storage_name) setval( run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX, os.path.join(output_storage_offset, '%s%s' % (sweep_name, contextid))) def _parse_input_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return (name, offset) input_loc = self.input_exists(run_settings) logger.debug('special_input_loc=%s' % input_loc) if input_loc: location = getval(run_settings, input_loc) input_storage_name, input_storage_offset = \ _parse_input_location(run_settings, location) setval( run_settings, '%s/platform/storage/input/platform_url' % django_settings.SCHEMA_PREFIX, input_storage_name) # store offsets setval( run_settings, '%s/platform/storage/input/offset' % django_settings.SCHEMA_PREFIX, input_storage_offset) # TODO: replace with scratch space computation platform space self.scratch_platform = '%s%s%s' % (manage.get_scratch_platform(), sweep_name, contextid) # mytardis if output_loc: try: self.experiment_id = int( getval( run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX)) except KeyError, ValueError: self.experiment_id = 0 try: curate_data = getval( run_settings, '%s/input/mytardis/curate_data' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: curate_data = False curate_data = False #TODO remove if curate_data: logger.debug('location=%s' % location) location = "%s%s" % (sweep_name, contextid) self.experiment_id = self.curate_data(run_settings, location, self.experiment_id) setval( run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id))
except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass v_map.update(starting_map) v_map.update(context) v_map['run_counter'] = 1 logger.debug("new v_map=%s" % v_map) put_file(values_url, json.dumps(v_map, indent=4)) # Set random numbers for subdirective logger.debug("run_settings=%s" % pformat(run_settings)) if rands: setval(run_settings, '%s/input/hrmc/iseed' % django_settings.SCHEMA_PREFIX, rands[i]) if input_loc: # Set revised input_location for subdirective setval( run_settings, input_loc, "%s/%s/%s" % (self.scratch_platform, SUBDIRECTIVE_DIR % { 'run_counter': str(run_counter) }, FIRST_ITERATION_DIR)) # Redirect input run_input_storage_name, run_input_storage_offset = \ _parse_input_location(run_settings, "local/%s%s/run%s/input_0" % (sweep_name, contextid, run_counter))
def process(self, run_settings): logger.debug('run_settings=%s' % run_settings) # Need to make copy because we pass on run_settings to sub connector # so any changes we make here to run_settings WILL be inherited def make_local_settings(run_settings): from copy import deepcopy local_settings = deepcopy(getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)) update(local_settings, run_settings, RMIT_SCHEMA + '/system/platform', # RMIT_SCHEMA + '/input/mytardis/experiment_id', # RMIT_SCHEMA + '/system/random_numbers', ) local_settings['bdp_username'] = getval( run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) return local_settings local_settings = make_local_settings(run_settings) logger.debug('local_settings=%s' % local_settings) setval(run_settings, '%s/platform/computation/platform_url' % RMIT_SCHEMA, getval(run_settings, '%s/input/system/compplatform/computation_platform' % RMIT_SCHEMA)) def _parse_output_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return name, offset contextid = int(getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)) logger.debug("contextid=%s" % contextid) sweep_name = self._get_sweep_name(run_settings) logger.debug("sweep_name=%s" % sweep_name) output_loc = self.output_exists(run_settings) location = "" if output_loc: location = getval(run_settings, output_loc) output_storage_name, output_storage_offset = \ _parse_output_location(run_settings, location) setval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA, output_storage_name) setval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA, os.path.join(output_storage_offset, '%s%s' % (sweep_name, contextid))) def _parse_input_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return (name, offset) input_loc = self.input_exists(run_settings) if input_loc: location = getval(run_settings, input_loc) input_storage_name, input_storage_offset = \ _parse_input_location(run_settings, location) setval(run_settings, '%s/platform/storage/input/platform_url' % RMIT_SCHEMA, input_storage_name) # store offsets setval(run_settings, '%s/platform/storage/input/offset' % RMIT_SCHEMA, input_storage_offset) # TODO: replace with scratch space computation platform space self.scratch_platform = '%s%s%s' % ( manage.get_scratch_platform(), sweep_name, contextid) # mytardis if output_loc: try: self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA)) except KeyError: self.experiment_id = 0 except ValueError: self.experiment_id = 0 try: curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA) except SettingNotFoundException: curate_data = False if curate_data: self.experiment_id = self.curate_data(run_settings, location, self.experiment_id) setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id)) # generate all variations map_text = getval(run_settings, '%s/input/sweep/sweep_map' % RMIT_SCHEMA) # map_text = run_settings[RMIT_SCHEMA + '/input/sweep']['sweep_map'] sweep_map = json.loads(map_text) logger.debug("sweep_map=%s" % pformat(sweep_map)) runs = _expand_variations(maps=[sweep_map], values={}) logger.debug("runs=%s" % runs) # Create random numbers if needed # TODO: move iseed out of hrmc into separate generic schema # to use on any sweepable connector and make this function # completely hrmc independent. rands = [] try: self.rand_index = getval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA) logger.debug("rand_index=%s" % self.rand_index) except SettingNotFoundException: pass else: # prep random seeds for each run based off original iseed # FIXME: inefficient for large random file # TODO, FIXME: this is potentially problematic if different # runs end up overlapping in the random numbers they utilise. # solution is to have separate random files per run or partition # big file up. try: num_url = getval(run_settings, "%s/system/random_numbers" % RMIT_SCHEMA) logger.debug('num_url=%s' % num_url) except SettingNotFoundException: pass else: try: local_settings['random_numbers'] = num_url rands = generate_rands(settings=local_settings, start_range=0, end_range=-1, num_required=len(runs), start_index=self.rand_index) logger.debug("rands=%s" % rands) except Exception, e: logger.debug('error') logger.error(e) raise
def output(self, run_settings): """ Output new runs_left value (including zero value) """ logger.debug("finished stage output") executing_procs = [x for x in self.current_processes if x['status'] != 'ready'] #nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes) nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes) #if len(self.finished_nodes) == 0 and self.failed_processes == 0: # nodes_working = 0 logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes)) #logger.debug("self.executed_procs=%s" % self.executed_procs) logger.debug("self.executed_procs=%s" % executing_procs) logger.debug("self.finished_nodes=%s" % self.finished_nodes) #FIXME: nodes_working can be negative # FIXME: possible race condition? # if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/run'): # run_settings['http://rmit.edu.au/schemas/stages/run'] = {} # #run_settings['http://rmit.edu.au/schemas/stages/run']['runs_left'] = nodes_working #run_settings['http://rmit.edu.au/schemas/stages/run']['error_nodes'] = nodes_working setvals(run_settings, { '%s/stages/run/runs_left' % RMIT_SCHEMA: nodes_working, '%s/stages/run/error_nodes' % RMIT_SCHEMA: nodes_working, }) if not nodes_working: self.finished_nodes = [] #messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1)) #run_settings['http://rmit.edu.au/schemas/stages/run']['finished_nodes'] = str(self.finished_nodes) setvals(run_settings, { '%s/stages/run/finished_nodes' % RMIT_SCHEMA: str(self.finished_nodes), '%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes), '%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes), '%s/stages/execute/executed_procs' % RMIT_SCHEMA: str(self.executed_procs) }) if self.failed_nodes: setval(run_settings, '%s/stages/create/failed_nodes' % RMIT_SCHEMA, self.failed_nodes) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/create', {})[u'failed_nodes'] = self.failed_nodes completed_procs = [x for x in self.executed_procs if x['status'] == 'completed'] # messages.info(run_settings, "%s: waiting (%s of %s processes done)" # % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes))) if self.procs_2b_rescheduled: setvals(run_settings, { '%s/stages/schedule/scheduled_started' % RMIT_SCHEMA: 0, '%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled }) # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'schedule_started'] = 0 # run_settings.setdefault( # 'http://rmit.edu.au/schemas/stages/schedule', # {})[u'procs_2b_rescheduled'] = self.procs_2b_rescheduled return run_settings
def output(self, run_settings): setval( run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id)) if not self.done_iterating: # trigger first of iteration corestages logger.debug("nonconvergence") setvals( run_settings, { '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: '[]', '%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 0, '%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 0 }) logger.debug('scheduled_nodes=%s' % getval( run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX)) try: delkey( run_settings, '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass try: delkey( run_settings, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass # run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] # del run['error_nodes'] #update_key('converged', False, context) setval( run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 0) # run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0 # delete_key('runs_left', context) # delete_key('error_nodes', context) # update_key('converged', False, context) else: logger.debug("convergence") # we are done, so trigger next stage outside of converge #update_key('converged', True, context) setval( run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 1) # we are done, so don't trigger iteration stages #update_key('criterion', self.criterion, context) setval(run_settings, '%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX, unicode(self.criterion)) # delete_key('error_nodes', context) #delete_key('transformed', context) try: delkey( run_settings, '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: pass self.id += 1 # update_key('id', self.id, context) setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX, self.id) return run_settings