def _get_output(self, local_settings, source_url): """ Retrieve the output from the task on the node """ logger.debug("get_output from %s" % source_url) computation_platform_url = local_settings['comp_platform_url'] bdp_username = local_settings['bdp_username'] comp_pltf_settings = manage.get_platform_settings( computation_platform_url, bdp_username) local_settings.update(comp_pltf_settings) encoded_s_url = storage.get_url_with_credentials( local_settings, source_url, is_relative_path=True, ip_address=local_settings['host']) (scheme, host, mypath, location, query_settings) = \ storage.parse_bdpurl(encoded_s_url) make_path = os.path.join(query_settings['root_path'], mypath) logger.debug("make_path=%s" % make_path) output_storage_url = local_settings['storeout_platform_url'] logger.debug("output_storage_url=%s" % output_storage_url) output_storage_settings = manage.get_platform_settings( output_storage_url, bdp_username) local_settings.update(output_storage_settings) logger.debug("output_storage_settings=%s" % output_storage_settings) dest_url = '%s://%s@%s/%s/make%s' % ( output_storage_settings['scheme'], output_storage_settings['type'], output_storage_settings['host'], local_settings['storeout_platform_offset'], str(local_settings['contextid'])) logger.debug("Transferring output from %s to %s" % (source_url, dest_url)) local_settings.update(output_storage_settings) encoded_d_url = storage.get_url_with_credentials( local_settings, dest_url) logger.debug("encoded_d_url=%s" % encoded_d_url) # FIXME: might want to turn on paramiko compress function #storage_files(encoded_d_url, exceptions=[]) # to speed up this transfer try: storage.copy_directories(encoded_s_url, encoded_d_url) except SSHException, e: logger.error(e) # FIXME: Could just exit, but need to flag that this data has not # been transferred. raise
def _get_output(self, local_settings, source_url): """ Retrieve the output from the task on the node """ logger.debug("get_output from %s" % source_url) computation_platform_url = local_settings['comp_platform_url'] bdp_username = local_settings['bdp_username'] comp_pltf_settings = manage.get_platform_settings( computation_platform_url, bdp_username) local_settings.update(comp_pltf_settings) encoded_s_url = storage.get_url_with_credentials( local_settings, source_url, is_relative_path=True, ip_address=local_settings['host']) (scheme, host, mypath, location, query_settings) = \ storage.parse_bdpurl(encoded_s_url) make_path = os.path.join(query_settings['root_path'], mypath) logger.debug("make_path=%s" % make_path) output_storage_url = local_settings['storeout_platform_url'] logger.debug("output_storage_url=%s" % output_storage_url) output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username) local_settings.update(output_storage_settings) logger.debug("output_storage_settings=%s" % output_storage_settings) dest_url = '%s://%s@%s/%s/make%s' % (output_storage_settings['scheme'], output_storage_settings['type'], output_storage_settings['host'], local_settings['storeout_platform_offset'], str(local_settings['contextid'])) logger.debug("Transferring output from %s to %s" % (source_url, dest_url)) local_settings.update(output_storage_settings) encoded_d_url = storage.get_url_with_credentials(local_settings, dest_url) logger.debug("encoded_d_url=%s" % encoded_d_url) # FIXME: might want to turn on paramiko compress function #storage_files(encoded_d_url, exceptions=[]) # to speed up this transfer try: storage.copy_directories(encoded_s_url, encoded_d_url) except SSHException, e: logger.error(e) # FIXME: Could just exit, but need to flag that this data has not # been transferred. raise
def copy_to_scratch_space(self, run_settings, local_settings): bdp_username = run_settings[ 'http://rmit.edu.au/schemas/bdp_userprofile']['username'] output_storage_url = run_settings[ 'http://rmit.edu.au/schemas/platform/storage/output'][ 'platform_url'] output_storage_settings = manage.get_platform_settings( output_storage_url, bdp_username) run_settings['http://rmit.edu.au/schemas/platform/storage/output'][ 'offset'] = self.output_loc_offset offset = run_settings[ 'http://rmit.edu.au/schemas/platform/storage/output']['offset'] self.job_dir = manage.get_job_dir(output_storage_settings, offset) iter_inputdir = os.path.join(self.job_dir, "input_0") logger.debug("iter_inputdir=%s" % iter_inputdir) input_location = run_settings[RMIT_SCHEMA + '/input/system']['input_location'] logger.debug("input_location=%s" % input_location) #todo: input location will evenatually be replaced by the scratch space that was used by the sweep #todo: the sweep will indicate the location of the scratch space in the run_settings #todo: add scheme (ssh) to inputlocation source_url = get_url_with_credentials(local_settings, input_location) logger.debug("source_url=%s" % source_url) destination_url = get_url_with_credentials( output_storage_settings, '%s://%s@%s' % (output_storage_settings['scheme'], output_storage_settings['type'], iter_inputdir), is_relative_path=False) logger.debug("destination_url=%s" % destination_url) storage.copy_directories(source_url, destination_url)
def post_mytardis_exp(run_settings, experiment_id, output_location, experiment_paramset=[]): # TODO: move into mytardis package? bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) try: mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA) except SettingNotFoundException: logger.error("mytardis_platform not set") return 0 mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) logger.debug(mytardis_settings) curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA) if curate_data: if mytardis_settings['mytardis_host']: def _get_exp_name_for_input(path): return str(os.sep.join(path.split(os.sep)[-1:])) ename = _get_exp_name_for_input(output_location) logger.debug("ename=%s" % ename) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, experiment_paramset=experiment_paramset, expname=ename) else: logger.warn("no mytardis host specified") else: logger.warn('Data curation is off') return experiment_id
def copy_to_scratch_space(self, run_settings, local_settings): bdp_username = run_settings['http://rmit.edu.au/schemas/bdp_userprofile']['username'] output_storage_url = run_settings['http://rmit.edu.au/schemas/platform/storage/output']['platform_url'] output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username) run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset'] = self.output_loc_offset offset = run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset'] self.job_dir = manage.get_job_dir(output_storage_settings, offset) iter_inputdir = os.path.join(self.job_dir, "input_0") logger.debug("iter_inputdir=%s" % iter_inputdir) input_location = run_settings[ RMIT_SCHEMA + '/input/system']['input_location'] logger.debug("input_location=%s" % input_location) #todo: input location will evenatually be replaced by the scratch space that was used by the sweep #todo: the sweep will indicate the location of the scratch space in the run_settings #todo: add scheme (ssh) to inputlocation source_url = get_url_with_credentials(local_settings, input_location) logger.debug("source_url=%s" % source_url) destination_url = get_url_with_credentials( output_storage_settings, '%s://%s@%s' % (output_storage_settings['scheme'], output_storage_settings['type'], iter_inputdir), is_relative_path=False) logger.debug("destination_url=%s" % destination_url) storage.copy_directories(source_url, destination_url)
def curate_data(self, run_settings, location, experiment_id): bdp_username = run_settings['http://rmit.edu.au/schemas/bdp_userprofile']['username'] curate_data = run_settings['http://rmit.edu.au/schemas/input/mytardis']['curate_data'] if curate_data: mytardis_url = run_settings['http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform'] mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) logger.debug(mytardis_settings) EXP_DATASET_NAME_SPLIT = 2 def _get_exp_name_for_input(path): return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:])) logger.debug("location=%s" % location) ename = _get_exp_name_for_input(location) logger.debug("ename=%s" % ename) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=[ mytardis.create_paramset("hrmcexp", []), mytardis.create_graph_paramset("expgraph", name="hrmcexp", graph_info={"axes":["iteration", "criterion"], "legends":["criterion"], "precision":[0, 2]}, value_dict={}, value_keys=[["hrmcdset/it", "hrmcdset/crit"]]) ]) else: logger.warn('Data curation is off') return experiment_id
def curate_data(self, run_settings, output_location, experiment_id): ''' Creates experiment in MyTardis ''' # Loading MyTardis credentials bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % SCHEMA_PREFIX) mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % SCHEMA_PREFIX) mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) def _get_experiment_name(path): ''' Return the name for MyTardis experiment e.g., if path='x/y/z', returns 'y/z' ''' return str(os.sep.join(path.split(os.sep)[-2:])) # Creates new experiment if experiment_id=0 # If experiment_id is non-zero, the experiment is updated experiment_id = mytardis.create_experiment( settings=mytardis_settings, # MyTardis credentials exp_id=experiment_id, expname=_get_experiment_name(output_location), # name of the experiment in MyTardis # metadata associated with the experiment # a list of parameter sets experiment_paramset=[ # a new blank parameter set conforming to schema 'remotemake' mytardis.create_paramset("remotemake", []), # a graph parameter set mytardis.create_graph_paramset("expgraph", # name of schema name="randexp1", # unique graph name graph_info={"axes":["x", "y"], "legends":["Random points"]}, # information about the graph value_dict={}, # values to be used in parent graphs if appropriate value_keys=[["randdset/x", "randdset/y"]]), # values from datasets to produce points in the graph ]) return experiment_id
def _upload_variations_inputs(settings, source_url_initial, values_map): bdp_username = settings['bdp_username'] logger.debug("source_url_initial=%s" % source_url_initial) encoded_s_url = storage.get_url_with_credentials(settings, source_url_initial) logger.debug("encoded_s_url=%s" % encoded_s_url) dest_url = _get_dest_bdp_url(settings) computation_platform_url = settings['comp_platform_url'] bdp_username = settings['bdp_username'] comp_pltf_settings = manage.get_platform_settings( computation_platform_url, bdp_username) settings.update(comp_pltf_settings) encoded_d_url = storage.get_url_with_credentials(settings, dest_url, is_relative_path=True, ip_address=settings['host']) storage.copy_directories(encoded_s_url, encoded_d_url) for content_fname, content in _instantiate_context( source_url_initial, settings, values_map).items(): content_url = storage.get_url_with_credentials( settings, os.path.join(dest_url, content_fname), is_relative_path=True, ip_address=settings['host']) logger.debug("content_url=%s" % content_url) storage.put_file(content_url, content.encode('utf-8')) _save_values(settings, dest_url, values_map) logger.debug("done input upload")
def create_experiment(self, run_settings, location, experiment_id): bdp_username = run_settings['%s/bdp_userprofile' % django_settings.SCHEMA_PREFIX]['username'] curate_data = run_settings['%s/input/mytardis' % django_settings.SCHEMA_PREFIX]['curate_data'] if curate_data: mytardis_url = run_settings['%s/input/mytardis' % django_settings.SCHEMA_PREFIX]['mytardis_platform'] mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) logger.debug(mytardis_settings) EXP_DATASET_NAME_SPLIT = 2 def _get_exp_name_for_input(path): return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:])) logger.debug("location=%s" % location) ename = _get_exp_name_for_input(location) logger.debug("ename=%s" % ename) experiment_paramset = [] self.load_metadata_builder(run_settings) if self.METADATA_BUILDER: experiment_paramset=self.METADATA_BUILDER.build_experiment_metadata(run_settings=run_settings) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=experiment_paramset) else: logger.warn('Data curation is off') return experiment_id
def process(self, run_settings): self.experiment_id = 0 local_settings = setup_settings(run_settings) self.experiment_id = local_settings['experiment_id'] messages.info(run_settings, "1: waiting for completion") logger.debug("settings=%s" % local_settings) try: self.runs_left = ast.literal_eval(getval(run_settings, '%s/stages/make/runs_left' % RMIT_SCHEMA)) except (ValueError, SettingNotFoundException): self.runs_left = [] # if self._exists(run_settings, # 'http://rmit.edu.au/schemas/stages/make', # u'runs_left'): # self.runs_left = ast.literal_eval( # run_settings['http://rmit.edu.au/schemas/stages/make'][u'runs_left']) # else: # self.runs_left = [] def _get_dest_bdp_url(local_settings): return "%s@%s" % ( "nci", os.path.join(local_settings['payload_destination'], str(local_settings['contextid']))) dest_url = _get_dest_bdp_url(local_settings) computation_platform_url = local_settings['comp_platform_url'] bdp_username = local_settings['bdp_username'] comp_pltf_settings = manage.get_platform_settings( computation_platform_url, bdp_username) local_settings.update(comp_pltf_settings) encoded_d_url = storage.get_url_with_credentials( local_settings, dest_url, is_relative_path=True, ip_address=local_settings['host']) (scheme, host, mypath, location, query_settings) = \ storage.parse_bdpurl(encoded_d_url) if self.runs_left: job_finished = self._job_finished( settings=local_settings, remote_path=dest_url) if not job_finished: return self._get_output(local_settings, dest_url) self.runs_left -= 1 if self.runs_left <= 0: messages.success(run_settings, "%s: finished" % (1)) logger.debug("processing finished")
def process(self, run_settings): self.experiment_id = 0 local_settings = setup_settings(run_settings) self.experiment_id = local_settings['experiment_id'] messages.info(run_settings, "1: waiting for completion") logger.debug("settings=%s" % local_settings) try: self.runs_left = ast.literal_eval( getval(run_settings, '%s/stages/make/runs_left' % RMIT_SCHEMA)) except (ValueError, SettingNotFoundException): self.runs_left = [] # if self._exists(run_settings, # 'http://rmit.edu.au/schemas/stages/make', # u'runs_left'): # self.runs_left = ast.literal_eval( # run_settings['http://rmit.edu.au/schemas/stages/make'][u'runs_left']) # else: # self.runs_left = [] def _get_dest_bdp_url(local_settings): return "%s@%s" % ("nci", os.path.join( local_settings['payload_destination'], str(local_settings['contextid']))) dest_url = _get_dest_bdp_url(local_settings) computation_platform_url = local_settings['comp_platform_url'] bdp_username = local_settings['bdp_username'] comp_pltf_settings = manage.get_platform_settings( computation_platform_url, bdp_username) local_settings.update(comp_pltf_settings) encoded_d_url = storage.get_url_with_credentials( local_settings, dest_url, is_relative_path=True, ip_address=local_settings['host']) (scheme, host, mypath, location, query_settings) = \ storage.parse_bdpurl(encoded_d_url) if self.runs_left: job_finished = self._job_finished(settings=local_settings, remote_path=dest_url) if not job_finished: return self._get_output(local_settings, dest_url) self.runs_left -= 1 if self.runs_left <= 0: messages.success(run_settings, "%s: finished" % (1)) logger.debug("processing finished")
def curate_data(self, run_settings, location, experiment_id): logger.debug("vasp curate_data") logger.debug('location=%s' % location) bdp_username = getval( run_settings, '%s/bdp_userprofile/username' % self.SCHEMA_PREFIX) mytardis_url = run_settings[ 'http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform'] mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) logger.debug(mytardis_settings) def _get_exp_name_for_input(path): return str(os.sep.join(path.split(os.sep)[-2:])) ename = _get_exp_name_for_input(location) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=[ mytardis.create_paramset("remotemake", []), mytardis.create_graph_paramset( "expgraph", name="makeexp1", graph_info={ "axes": ["num_kp", "energy"], "legends": ["TOTEN"] }, value_dict={}, value_keys=[["makedset/num_kp", "makedset/toten"]]), mytardis.create_graph_paramset( "expgraph", name="makeexp2", graph_info={ "axes": ["encut", "energy"], "legends": ["TOTEN"] }, value_dict={}, value_keys=[["makedset/encut", "makedset/toten"]]), mytardis.create_graph_paramset( "expgraph", name="makeexp3", graph_info={ "axes": ["num_kp", "encut", "TOTEN"], "legends": ["TOTEN"] }, value_dict={}, value_keys=[[ "makedset/num_kp", "makedset/encut", "makedset/toten" ]]), ]) return experiment_id
def curate_data(self, run_settings, location, experiment_id): logger.debug("vasp durate_data") try: subdirective = getval(run_settings, '%s/stages/sweep/directive' % SCHEMA_PREFIX) except SettingNotFoundException: logger.warn("cannot find subdirective name") subdirective = '' if subdirective == "vasp": bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % SCHEMA_PREFIX) mytardis_url = run_settings['http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform'] mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) logger.debug(mytardis_settings) def _get_exp_name_for_input(path): return str(os.sep.join(path.split(os.sep)[-2:])) ename = _get_exp_name_for_input(location) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=[ mytardis.create_paramset("remotemake", []), mytardis.create_graph_paramset("expgraph", name="makeexp1", graph_info={"axes":["num_kp", "energy"], "legends":["TOTEN"]}, value_dict={}, value_keys=[["makedset/num_kp", "makedset/toten"]]), mytardis.create_graph_paramset("expgraph", name="makeexp2", graph_info={"axes":["encut", "energy"], "legends":["TOTEN"]}, value_dict={}, value_keys=[["makedset/encut", "makedset/toten"]]), mytardis.create_graph_paramset("expgraph", name="makeexp3", graph_info={"axes":["num_kp", "encut", "TOTEN"], "legends":["TOTEN"]}, value_dict={}, value_keys=[["makedset/num_kp", "makedset/encut", "makedset/toten"]]), ]) else: logger.warn("cannot find subdirective name") return experiment_id
def curate_data(self, run_settings, output_location, experiment_id): ''' Creates experiment in MyTardis ''' # Loading MyTardis credentials bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % SCHEMA_PREFIX) mytardis_url = getval( run_settings, '%s/input/mytardis/mytardis_platform' % SCHEMA_PREFIX) mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) def _get_experiment_name(path): ''' Return the name for MyTardis experiment e.g., if path='x/y/z', returns 'y/z' ''' return str(os.sep.join(path.split(os.sep)[-2:])) # Creates new experiment if experiment_id=0 # If experiment_id is non-zero, the experiment is updated experiment_id = mytardis.create_experiment( settings=mytardis_settings, # MyTardis credentials exp_id=experiment_id, expname=_get_experiment_name( output_location), # name of the experiment in MyTardis # metadata associated with the experiment # a list of parameter sets experiment_paramset=[ # a new blank parameter set conforming to schema 'remotemake' mytardis.create_paramset("remotemake", []), # a graph parameter set mytardis.create_graph_paramset( "expgraph", # name of schema name="randexp1", # unique graph name graph_info={ "axes": ["x", "y"], "legends": ["Random points"] }, # information about the graph value_dict= {}, # values to be used in parent graphs if appropriate value_keys=[[ "randdset/x", "randdset/y" ]]), # values from datasets to produce points in the graph ]) return experiment_id
def copy_to_scratch_space(self, run_settings, local_settings, result_offset): bdp_username = run_settings['%s/bdp_userprofile' % django_settings.SCHEMA_PREFIX]['username'] output_storage_url = run_settings['%s/platform/storage/output' % django_settings.SCHEMA_PREFIX]['platform_url'] output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username) run_settings['%s/platform/storage/output' % django_settings.SCHEMA_PREFIX]['offset'] = self.output_loc_offset offset = run_settings['%s/platform/storage/output' % django_settings.SCHEMA_PREFIX]['offset'] self.job_dir = manage.get_job_dir(output_storage_settings, offset) iter_inputdir = os.path.join(self.job_dir, result_offset) logger.debug("iter_inputdir=%s" % iter_inputdir) input_storage_settings = self.get_platform_settings(run_settings, '%s/platform/storage/input' % django_settings.SCHEMA_PREFIX) #input_location = run_settings[django_settings.SCHEMA_PREFIX + '/input/system']['input_location'] try: input_location = getval(run_settings, django_settings.SCHEMA_PREFIX + '/input/system/input_location') except SettingNotFoundException: try: input_location = getval(run_settings, django_settings.SCHEMA_PREFIX + '/input/location/input_location') except: input_location = getval(run_settings, django_settings.SCHEMA_PREFIX + '/input/location/input/input_location') logger.debug("input_location=%s" % input_location) #todo: input location will evenatually be replaced by the scratch space that was used by the sweep #todo: the sweep will indicate the location of the scratch space in the run_settings #todo: add scheme (ssh) to inputlocation #source_url = get_url_with_credentials(local_settings, input_location) input_offset = run_settings['%s/platform/storage/input' % django_settings.SCHEMA_PREFIX]['offset'] input_url = "%s://%s@%s/%s" % (input_storage_settings['scheme'], input_storage_settings['type'], input_storage_settings['host'], input_offset) source_url = get_url_with_credentials( input_storage_settings, input_url, is_relative_path=False) logger.debug("source_url=%s" % source_url) destination_url = get_url_with_credentials( output_storage_settings, '%s://%s@%s' % (output_storage_settings['scheme'], output_storage_settings['type'], iter_inputdir), is_relative_path=False) logger.debug("destination_url=%s" % destination_url) storage.copy_directories(source_url, destination_url)
def curate_data(self, run_settings, location, experiment_id): bdp_username = run_settings[ 'http://rmit.edu.au/schemas/bdp_userprofile']['username'] curate_data = run_settings[ 'http://rmit.edu.au/schemas/input/mytardis']['curate_data'] if curate_data: mytardis_url = run_settings[ 'http://rmit.edu.au/schemas/input/mytardis'][ 'mytardis_platform'] mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) logger.debug(mytardis_settings) EXP_DATASET_NAME_SPLIT = 2 def _get_exp_name_for_input(path): return str( os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:])) logger.debug("location=%s" % location) ename = _get_exp_name_for_input(location) logger.debug("ename=%s" % ename) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=[ mytardis.create_paramset("hrmcexp", []), mytardis.create_graph_paramset( "expgraph", name="hrmcexp", graph_info={ "axes": ["iteration", "criterion"], "legends": ["criterion"], "precision": [0, 2] }, value_dict={}, value_keys=[["hrmcdset/it", "hrmcdset/crit"]]) ]) else: logger.warn('Data curation is off') return experiment_id
def process(self, run_settings): """ perform the stage operation """ bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) # bdp_username = run_settings[RMIT_SCHEMA + '/bdp_userprofile']['username'] logger.debug("bdp_username=%s" % bdp_username) input_storage_url = getval(run_settings, '%s/platform/storage/input/platform_url' % RMIT_SCHEMA) # input_storage_url = run_settings[ # RMIT_SCHEMA + '/platform/storage/input']['platform_url'] logger.debug("input_storage_url=%s" % input_storage_url) input_storage_settings = manage.get_platform_settings( input_storage_url, bdp_username) logger.debug("input_storage_settings=%s" % pformat(input_storage_settings)) input_offset = getval(run_settings, '%s/platform/storage/input/offset' % RMIT_SCHEMA) # input_offset = run_settings[RMIT_SCHEMA + "/platform/storage/input"]['offset'] logger.debug("input_offset=%s" % pformat(input_offset)) input_prefix = '%s://%s@' % (input_storage_settings['scheme'], input_storage_settings['type']) map_initial_location = "%s/%s/initial" % (input_prefix, input_offset) logger.debug("map_initial_location=%s" % map_initial_location) local_settings = setup_settings(run_settings) logger.debug("local_settings=%s" % local_settings) values_map = _load_values_map(local_settings, map_initial_location) logger.debug("values_map=%s" % values_map) _upload_variations_inputs( local_settings, map_initial_location, values_map) _upload_payload(local_settings, local_settings['payload_source'], values_map) messages.info(run_settings, "1: upload done")
def create_experiment(self, run_settings, location, experiment_id): bdp_username = run_settings['%s/bdp_userprofile' % django_settings.SCHEMA_PREFIX]['username'] curate_data = run_settings[ '%s/input/mytardis' % django_settings.SCHEMA_PREFIX]['curate_data'] if curate_data: mytardis_url = run_settings[ '%s/input/mytardis' % django_settings.SCHEMA_PREFIX]['mytardis_platform'] mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) logger.debug(mytardis_settings) EXP_DATASET_NAME_SPLIT = 2 def _get_exp_name_for_input(path): return str( os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:])) logger.debug("location=%s" % location) ename = _get_exp_name_for_input(location) logger.debug("ename=%s" % ename) experiment_paramset = [] self.load_metadata_builder(run_settings) if self.METADATA_BUILDER: experiment_paramset = self.METADATA_BUILDER.build_experiment_metadata( run_settings=run_settings) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=experiment_paramset) else: logger.warn('Data curation is off') return experiment_id
def curate_data(self, run_settings, location, experiment_id): logger.debug("vasp curate_data") logger.debug('location=%s' % location) bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % self.SCHEMA_PREFIX) mytardis_url = run_settings['http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform'] mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) logger.debug(mytardis_settings) def _get_exp_name_for_input(path): return str(os.sep.join(path.split(os.sep)[-2:])) ename = _get_exp_name_for_input(location) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=[ mytardis.create_paramset("remotemake", []), mytardis.create_graph_paramset("expgraph", name="makeexp1", graph_info={"axes":["num_kp", "energy"], "legends":["TOTEN"]}, value_dict={}, value_keys=[["makedset/num_kp", "makedset/toten"]]), mytardis.create_graph_paramset("expgraph", name="makeexp2", graph_info={"axes":["encut", "energy"], "legends":["TOTEN"]}, value_dict={}, value_keys=[["makedset/encut", "makedset/toten"]]), mytardis.create_graph_paramset("expgraph", name="makeexp3", graph_info={"axes":["num_kp", "encut", "TOTEN"], "legends":["TOTEN"]}, value_dict={}, value_keys=[["makedset/num_kp", "makedset/encut", "makedset/toten"]]), ]) return experiment_id
def process(self, run_settings): settings = setup_settings(run_settings) messages.info(run_settings, "1: execute starting") def _get_dest_bdp_url(settings): return "%s@%s" % ( "nci", os.path.join(settings['payload_destination'], str(settings['contextid']))) dest_url = _get_dest_bdp_url(settings) computation_platform_url = settings['comp_platform_url'] bdp_username = settings['bdp_username'] comp_pltf_settings = manage.get_platform_settings( computation_platform_url, bdp_username) logger.debug("comp_pltf_settings=%s" % pformat(comp_pltf_settings)) settings.update(comp_pltf_settings) encoded_d_url = storage.get_url_with_credentials( settings, dest_url, is_relative_path=True, ip_address=settings['host']) (scheme, host, mypath, location, query_settings) = \ storage.parse_bdpurl(encoded_d_url) stderr = '' try: ssh = open_connection( ip_address=settings['host'], settings=settings) (command_out, stderr) = compute.run_make(ssh, (os.path.join( query_settings['root_path'], mypath)), 'startrun') except Exception, e: logger.error(e) raise
def process(self, run_settings): # self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA) bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA) output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username) logger.debug("output_storage_settings=%s" % output_storage_settings) output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA) self.job_dir = manage.get_job_dir(output_storage_settings, offset) try: self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA)) self.output_dir = os.path.join(os.path.join(self.job_dir, "output_%s" % self.id)) self.input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % self.id)) self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % (self.id + 1))) except (SettingNotFoundException, ValueError): # FIXME: Not clear that this a valid path through stages self.output_dir = os.path.join(os.path.join(self.job_dir, "output")) self.output_dir = os.path.join(os.path.join(self.job_dir, "input")) self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_1")) logger.debug('self.output_dir=%s' % self.output_dir) try: self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA)) except SettingNotFoundException: self.experiment_id = 0 except ValueError: self.experiment_id = 0 output_url = get_url_with_credentials( output_storage_settings, output_prefix + self.output_dir, is_relative_path=False) # (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url) # fsys = storage.get_filesystem(output_url) # node_output_dirs, _ = fsys.listdir(mypath) # logger.debug("node_output_dirs=%s" % node_output_dirs) self.process_outputs(run_settings, self.job_dir, output_url, output_storage_settings, offset) # logger.debug("output_url=%s" % output_url) # # Should this be output_dir or root of remotesys? # (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url) # fsys = storage.get_filesystem(output_url) # logger.debug("fsys=%s" % fsys) # logger.debug("mypath=%s" % mypath) # node_output_dirs, _ = fsys.listdir(mypath) # logger.debug("node_output_dirs=%s" % node_output_dirs) # self.audit = "" # outputs = [] # Node_info = namedtuple('Node_info', # ['dir', 'index', 'number', 'criterion']) # # gather node_infos # for node_output_dir in node_output_dirs: # base_fname = "HRMC.inp" # try: # values_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.output_dir, node_output_dir, # '%s_values' % base_fname), is_relative_path=False) # values_content = storage.get_file(values_url) # logger.debug("values_file=%s" % values_url) # except IOError: # logger.warn("no values file found") # values_map = {} # else: # values_map = dict(json.loads(values_content)) # criterion = self.compute_psd_criterion( # node_output_dir, fsys, # output_storage_settings) # #criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dir, fs,) # logger.debug("criterion=%s" % criterion) # index = 0 # FIXME: as node_output_dirs in particular order, then index is not useful. # outputs.append(Node_info(dir=node_output_dir, # index=index, number=values_map['run_counter'], criterion=criterion)) # outputs.sort(key=lambda x: int(x.criterion)) # logger.debug("outputs=%s" % outputs) # logger.debug('threshold=%s' % self.threshold) # total_picks = 1 # if len(self.threshold) > 1: # for i in self.threshold: # total_picks *= self.threshold[i] # else: # total_picks = self.threshold[0] # if not outputs: # logger.error("no ouput found for this iteration") # return # for index in range(0, total_picks): # Node_info = outputs[index] # logger.debug("node_info.dir=%s" % Node_info.dir) # logger.debug("Node_info=%s" % str(Node_info)) # self.new_input_node_dir = os.path.join(self.new_input_dir, # Node_info.dir) # logger.debug("New input node dir %s" % self.new_input_node_dir) # # Move all existing domain input files unchanged to next input directory # for f in self.DOMAIN_INPUT_FILES: # source_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.output_dir, Node_info.dir, f), is_relative_path=False) # dest_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_node_dir, f), # is_relative_path=False) # logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) # content = storage.get_file(source_url) # logger.debug('content collected') # storage.put_file(dest_url, content) # logger.debug('put successfully') # logger.debug('put file successfully') # pattern = "*_values" # self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir), # self.new_input_node_dir, pattern, # output_storage_settings) # pattern = "*_template" # self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir), # self.new_input_node_dir, pattern, # output_storage_settings) # # NB: Converge stage triggers based on criterion value from audit. # info = "Run %s preserved (error %s)\n" % (Node_info.number, Node_info.criterion) # audit_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_node_dir, 'audit.txt'), is_relative_path=False) # storage.put_file(audit_url, info) # logger.debug("audit=%s" % info) # self.audit += info # # move xyz_final.xyz to initial.xyz # source_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.output_dir, Node_info.dir, "xyz_final.xyz"), is_relative_path=False) # dest_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_node_dir, 'input_initial.xyz'), is_relative_path=False) # content = storage.get_file(source_url) # storage.put_file(dest_url, content) # self.audit += "spawning diamond runs\n" # audit_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_dir, 'audit.txt'), is_relative_path=False) # storage.put_file(audit_url, self.audit) # curate dataset into mytardis try: curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA) except SettingNotFoundException: curate_data = 0 if curate_data: mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA) mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) all_settings = dict(mytardis_settings) all_settings.update(output_storage_settings) self.experiment_id = self.curate_dataset(run_settings, self.experiment_id, self.job_dir, output_url, all_settings) else: logger.warn('Data curation is off')
def process(self, run_settings): try: id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX)) except (SettingNotFoundException, ValueError): id = 0 messages.info(run_settings, '%d: transforming' % (id+1)) # self.contextid = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX) bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX) output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX) output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username) logger.debug("output_storage_settings=%s" % output_storage_settings) output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) offset = getval(run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX) self.job_dir = manage.get_job_dir(output_storage_settings, offset) try: self.id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX)) self.output_dir = os.path.join(os.path.join(self.job_dir, "output_%s" % self.id)) self.input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % self.id)) self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % (self.id + 1))) except (SettingNotFoundException, ValueError): # FIXME: Not clear that this a valid path through stages self.output_dir = os.path.join(os.path.join(self.job_dir, "output")) self.output_dir = os.path.join(os.path.join(self.job_dir, "input")) self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_1")) logger.debug('self.output_dir=%s' % self.output_dir) try: self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX)) except SettingNotFoundException: self.experiment_id = 0 except ValueError: self.experiment_id = 0 output_url = get_url_with_credentials( output_storage_settings, output_prefix + self.output_dir, is_relative_path=False) # (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url) # fsys = storage.get_filesystem(output_url) # node_output_dirs, _ = fsys.listdir(mypath) # logger.debug("node_output_dirs=%s" % node_output_dirs) outputs = self.process_outputs(run_settings, self.job_dir, output_url, output_storage_settings, offset) try: curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: curate_data = 0 if curate_data: mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % django_settings.SCHEMA_PREFIX) mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) all_settings = dict(mytardis_settings) all_settings.update(output_storage_settings) all_settings['contextid'] = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX) try: mytardis_platform = jobs.safe_import('chiminey.platform.mytardis.MyTardisPlatform', [], {}) logger.debug('self_outpus=%s' % outputs) self.experiment_id = mytardis_platform.create_dataset_for_intermediate_output(run_settings, self.experiment_id, self.job_dir, output_url, all_settings, outputs=outputs) except ImproperlyConfigured as e: logger.error("Cannot load mytardis platform hook %s" % e) else: logger.warn('Data curation is off')
def process(self, run_settings): try: id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA)) except (SettingNotFoundException, ValueError): id = 0 messages.info(run_settings, '%d: converging' % (id+1)) def retrieve_local_settings(run_settings, local_settings): update(local_settings, run_settings # '%s/stages/setup/payload_source' % RMIT_SCHEMA, # '%s/stages/setup/payload_destination' % RMIT_SCHEMA, # '%s/system/platform' % RMIT_SCHEMA, # # '%s/stages/create/custom_prompt' % RMIT_SCHEMA, # # '%s/stages/create/cloud_sleep_interval' % RMIT_SCHEMA, # # '%s/stages/create/created_nodes' % RMIT_SCHEMA, # '%s/stages/run/payload_cloud_dirname' % RMIT_SCHEMA, # '%s/system/max_seed_int' % RMIT_SCHEMA, # '%s/stages/run/compile_file' % RMIT_SCHEMA, # '%s/stages/run/retry_attempts' % RMIT_SCHEMA, # '%s/input/system/cloud/number_vm_instances' % RMIT_SCHEMA, # '%s/input/hrmc/iseed' % RMIT_SCHEMA, # '%s/input/hrmc/optimisation_scheme' % RMIT_SCHEMA, # '%s/input/hrmc/threshold' % RMIT_SCHEMA, ) local_settings['bdp_username'] = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS) retrieve_local_settings(run_settings, local_settings) bdp_username = local_settings['bdp_username'] # get output output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA) output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username) output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA) job_dir = manage.get_job_dir(output_storage_settings, offset) # get mytardis #mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA) #mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) # setup new paths try: self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA)) self.output_dir = os.path.join(job_dir, "output_%d" % self.id) self.iter_inputdir = os.path.join(job_dir, "input_%d" % (self.id + 1)) #self.new_iter_inputdir = "input_%d" % (self.id + 1) except (SettingNotFoundException, ValueError): self.output_dir = os.path.join(job_dir, "output") self.iter_inputdir = os.path.join(job_dir, "input") self.id = 0 logger.debug('output_dir=%s iter_inputdir=%s' % (self.output_dir, self.iter_inputdir)) try: self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA)) except SettingNotFoundException: self.experiment_id = 0 except ValueError: self.experiment_id = 0 inputdir_url = get_url_with_credentials(output_storage_settings, output_prefix + self.iter_inputdir, is_relative_path=False) logger.debug('input_dir_url=%s' % inputdir_url) # (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(inputdir_url) # fsys = storage.get_filesystem(inputdir_url) # logger.debug('mypath=%s' % mypath) # input_dirs, _ = fsys.listdir(mypath) # logger.debug('input_dirs=%s' % input_dirs) (self.done_iterating, self.criterion) = self.process_outputs(run_settings, job_dir, inputdir_url, output_storage_settings) if self.done_iterating: logger.debug("Total Iterations: %d" % self.id) # output_prefix = '%s://%s@' % (output_storage_settings['scheme'], # output_storage_settings['type']) # new_output_dir = os.path.join(base_dir, 'output') output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) # get source url iter_output_dir = os.path.join(os.path.join(job_dir, "output_%s" % self.id)) source_url = "%s%s" % (output_prefix, iter_output_dir) # get dest url new_output_dir = os.path.join(job_dir, 'output') dest_url = "%s%s" % (output_prefix, new_output_dir) source_url = get_url_with_credentials(output_storage_settings, output_prefix + os.path.join(iter_output_dir), is_relative_path=False) dest_url = get_url_with_credentials(output_storage_settings, output_prefix + os.path.join(new_output_dir), is_relative_path=False) storage.copy_directories(source_url, dest_url) # curate try: curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA) except SettingNotFoundException: curate_data = 0 if curate_data: mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA) mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username) all_settings = dict(mytardis_settings) all_settings.update(output_storage_settings) logger.debug("source_url=%s" % source_url) logger.debug("dest_url=%s" % dest_url) logger.debug("job_dir=%s" % job_dir) self.experiment_id = self.curate_dataset(run_settings, self.experiment_id, job_dir, dest_url, all_settings) else: logger.warn('Data curation is off')
self.rand_index = 42 logger.debug(e) logger.debug("processing execute stage") local_settings = getvals( run_settings, models.UserProfile.PROFILE_SCHEMA_NS) self.set_execute_settings(run_settings, local_settings) self.contextid = getval( run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX) # NB: Don't catch SettingNotFoundException because we can't recover # run_settings['%s/system' % django_settings.SCHEMA_PREFIX][u'contextid'] logger.debug('contextid=%s' % self.contextid) output_storage_url = getval( run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX) output_storage_settings = manage.get_platform_settings( output_storage_url, local_settings['bdp_username']) offset = getval( run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX) self.job_dir = manage.get_job_dir(output_storage_settings, offset) # TODO: we assume initial input is in "%s/input_0" % self.job_dir # in configure stage we could copy initial data in 'input_location' # into this location try: self.id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX)) self.iter_inputdir = os.path.join( self.job_dir, "input_%s" % self.id) except (SettingNotFoundException, ValueError): self.id = 0 self.iter_inputdir = os.path.join(self.job_dir, "input_location") messages.info(run_settings, "%s: Executing" % (self.id + 1))
except SettingNotFoundException, e: self.rand_index = 42 logger.debug(e) logger.debug("processing execute stage") local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS) #self.retrieve_boto_settings(run_settings, local_settings) self.set_execute_settings(run_settings, local_settings) self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA) # NB: Don't catch SettingNotFoundException because we can't recover # run_settings['http://rmit.edu.au/schemas/system'][u'contextid'] output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA) output_storage_settings = manage.get_platform_settings(output_storage_url, local_settings['bdp_username']) offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA) # offset = run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset'] self.job_dir = manage.get_job_dir(output_storage_settings, offset) # TODO: we assume initial input is in "%s/input_0" % self.job_dir # in configure stage we could copy initial data in 'input_location' into this location try: self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA)) self.iter_inputdir = os.path.join(self.job_dir, "input_%s" % self.id) except (SettingNotFoundException, ValueError): self.id = 0 self.iter_inputdir = os.path.join(self.job_dir, "input_location") messages.info(run_settings, "%s: execute" % (self.id + 1)) logger.debug("id = %s" % self.id) try:
def curate_data(self, run_settings, location, experiment_id): logger.debug("vasp durate_data") try: subdirective = getval(run_settings, '%s/stages/sweep/directive' % SCHEMA_PREFIX) except SettingNotFoundException: logger.warn("cannot find subdirective name") subdirective = '' if subdirective == "vasp": bdp_username = getval( run_settings, '%s/bdp_userprofile/username' % SCHEMA_PREFIX) mytardis_url = run_settings[ 'http://rmit.edu.au/schemas/input/mytardis'][ 'mytardis_platform'] mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) logger.debug(mytardis_settings) def _get_exp_name_for_input(path): return str(os.sep.join(path.split(os.sep)[-2:])) ename = _get_exp_name_for_input(location) experiment_id = mytardis.create_experiment( settings=mytardis_settings, exp_id=experiment_id, expname=ename, experiment_paramset=[ mytardis.create_paramset("remotemake", []), mytardis.create_graph_paramset( "expgraph", name="makeexp1", graph_info={ "axes": ["num_kp", "energy"], "legends": ["TOTEN"] }, value_dict={}, value_keys=[["makedset/num_kp", "makedset/toten"]]), mytardis.create_graph_paramset( "expgraph", name="makeexp2", graph_info={ "axes": ["encut", "energy"], "legends": ["TOTEN"] }, value_dict={}, value_keys=[["makedset/encut", "makedset/toten"]]), mytardis.create_graph_paramset( "expgraph", name="makeexp3", graph_info={ "axes": ["num_kp", "encut", "TOTEN"], "legends": ["TOTEN"] }, value_dict={}, value_keys=[[ "makedset/num_kp", "makedset/encut", "makedset/toten" ]]), ]) else: logger.warn("cannot find subdirective name") return experiment_id
def process(self, run_settings): # self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA) bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) output_storage_url = getval( run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA) output_storage_settings = manage.get_platform_settings( output_storage_url, bdp_username) logger.debug("output_storage_settings=%s" % output_storage_settings) output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA) self.job_dir = manage.get_job_dir(output_storage_settings, offset) try: self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA)) self.output_dir = os.path.join( os.path.join(self.job_dir, "output_%s" % self.id)) self.input_dir = os.path.join( os.path.join(self.job_dir, "input_%d" % self.id)) self.new_input_dir = os.path.join( os.path.join(self.job_dir, "input_%d" % (self.id + 1))) except (SettingNotFoundException, ValueError): # FIXME: Not clear that this a valid path through stages self.output_dir = os.path.join(os.path.join( self.job_dir, "output")) self.output_dir = os.path.join(os.path.join(self.job_dir, "input")) self.new_input_dir = os.path.join( os.path.join(self.job_dir, "input_1")) logger.debug('self.output_dir=%s' % self.output_dir) try: self.experiment_id = int( getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA)) except SettingNotFoundException: self.experiment_id = 0 except ValueError: self.experiment_id = 0 output_url = get_url_with_credentials(output_storage_settings, output_prefix + self.output_dir, is_relative_path=False) # (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url) # fsys = storage.get_filesystem(output_url) # node_output_dirs, _ = fsys.listdir(mypath) # logger.debug("node_output_dirs=%s" % node_output_dirs) self.process_outputs(run_settings, self.job_dir, output_url, output_storage_settings, offset) # logger.debug("output_url=%s" % output_url) # # Should this be output_dir or root of remotesys? # (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url) # fsys = storage.get_filesystem(output_url) # logger.debug("fsys=%s" % fsys) # logger.debug("mypath=%s" % mypath) # node_output_dirs, _ = fsys.listdir(mypath) # logger.debug("node_output_dirs=%s" % node_output_dirs) # self.audit = "" # outputs = [] # Node_info = namedtuple('Node_info', # ['dir', 'index', 'number', 'criterion']) # # gather node_infos # for node_output_dir in node_output_dirs: # base_fname = "HRMC.inp" # try: # values_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.output_dir, node_output_dir, # '%s_values' % base_fname), is_relative_path=False) # values_content = storage.get_file(values_url) # logger.debug("values_file=%s" % values_url) # except IOError: # logger.warn("no values file found") # values_map = {} # else: # values_map = dict(json.loads(values_content)) # criterion = self.compute_psd_criterion( # node_output_dir, fsys, # output_storage_settings) # #criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dir, fs,) # logger.debug("criterion=%s" % criterion) # index = 0 # FIXME: as node_output_dirs in particular order, then index is not useful. # outputs.append(Node_info(dir=node_output_dir, # index=index, number=values_map['run_counter'], criterion=criterion)) # outputs.sort(key=lambda x: int(x.criterion)) # logger.debug("outputs=%s" % outputs) # logger.debug('threshold=%s' % self.threshold) # total_picks = 1 # if len(self.threshold) > 1: # for i in self.threshold: # total_picks *= self.threshold[i] # else: # total_picks = self.threshold[0] # if not outputs: # logger.error("no ouput found for this iteration") # return # for index in range(0, total_picks): # Node_info = outputs[index] # logger.debug("node_info.dir=%s" % Node_info.dir) # logger.debug("Node_info=%s" % str(Node_info)) # self.new_input_node_dir = os.path.join(self.new_input_dir, # Node_info.dir) # logger.debug("New input node dir %s" % self.new_input_node_dir) # # Move all existing domain input files unchanged to next input directory # for f in self.DOMAIN_INPUT_FILES: # source_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.output_dir, Node_info.dir, f), is_relative_path=False) # dest_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_node_dir, f), # is_relative_path=False) # logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) # content = storage.get_file(source_url) # logger.debug('content collected') # storage.put_file(dest_url, content) # logger.debug('put successfully') # logger.debug('put file successfully') # pattern = "*_values" # self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir), # self.new_input_node_dir, pattern, # output_storage_settings) # pattern = "*_template" # self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir), # self.new_input_node_dir, pattern, # output_storage_settings) # # NB: Converge stage triggers based on criterion value from audit. # info = "Run %s preserved (error %s)\n" % (Node_info.number, Node_info.criterion) # audit_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_node_dir, 'audit.txt'), is_relative_path=False) # storage.put_file(audit_url, info) # logger.debug("audit=%s" % info) # self.audit += info # # move xyz_final.xyz to initial.xyz # source_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.output_dir, Node_info.dir, "xyz_final.xyz"), is_relative_path=False) # dest_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_node_dir, 'input_initial.xyz'), is_relative_path=False) # content = storage.get_file(source_url) # storage.put_file(dest_url, content) # self.audit += "spawning diamond runs\n" # audit_url = get_url_with_credentials( # output_storage_settings, # output_prefix + os.path.join(self.new_input_dir, 'audit.txt'), is_relative_path=False) # storage.put_file(audit_url, self.audit) # curate dataset into mytardis try: curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA) except SettingNotFoundException: curate_data = 0 if curate_data: mytardis_url = getval( run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA) mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) all_settings = dict(mytardis_settings) all_settings.update(output_storage_settings) self.experiment_id = self.curate_dataset(run_settings, self.experiment_id, self.job_dir, output_url, all_settings) else: logger.warn('Data curation is off')
def process(self, run_settings): try: id = int( getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX)) except (SettingNotFoundException, ValueError): id = 0 messages.info(run_settings, '%d: transforming' % (id + 1)) # self.contextid = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX) bdp_username = getval( run_settings, '%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX) output_storage_url = getval( run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX) output_storage_settings = manage.get_platform_settings( output_storage_url, bdp_username) logger.debug("output_storage_settings=%s" % output_storage_settings) output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) offset = getval( run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX) self.job_dir = manage.get_job_dir(output_storage_settings, offset) try: self.id = int( getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX)) self.output_dir = os.path.join( os.path.join(self.job_dir, "output_%s" % self.id)) self.input_dir = os.path.join( os.path.join(self.job_dir, "input_%d" % self.id)) self.new_input_dir = os.path.join( os.path.join(self.job_dir, "input_%d" % (self.id + 1))) except (SettingNotFoundException, ValueError): # FIXME: Not clear that this a valid path through stages self.output_dir = os.path.join(os.path.join( self.job_dir, "output")) self.output_dir = os.path.join(os.path.join(self.job_dir, "input")) self.new_input_dir = os.path.join( os.path.join(self.job_dir, "input_1")) logger.debug('self.output_dir=%s' % self.output_dir) try: self.experiment_id = int( getval( run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX)) except SettingNotFoundException: self.experiment_id = 0 except ValueError: self.experiment_id = 0 output_url = get_url_with_credentials(output_storage_settings, output_prefix + self.output_dir, is_relative_path=False) # (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url) # fsys = storage.get_filesystem(output_url) # node_output_dirs, _ = fsys.listdir(mypath) # logger.debug("node_output_dirs=%s" % node_output_dirs) outputs = self.process_outputs(run_settings, self.job_dir, output_url, output_storage_settings, offset) try: curate_data = getval( run_settings, '%s/input/mytardis/curate_data' % django_settings.SCHEMA_PREFIX) except SettingNotFoundException: curate_data = 0 if curate_data: mytardis_url = getval( run_settings, '%s/input/mytardis/mytardis_platform' % django_settings.SCHEMA_PREFIX) mytardis_settings = manage.get_platform_settings( mytardis_url, bdp_username) all_settings = dict(mytardis_settings) all_settings.update(output_storage_settings) all_settings['contextid'] = getval( run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX) try: mytardis_platform = jobs.safe_import( 'chiminey.platform.mytardis.MyTardisPlatform', [], {}) logger.debug('self_outpus=%s' % outputs) self.experiment_id = mytardis_platform.create_dataset_for_intermediate_output( run_settings, self.experiment_id, self.job_dir, output_url, all_settings, outputs=outputs) except ImproperlyConfigured as e: logger.error("Cannot load mytardis platform hook %s" % e) else: logger.warn('Data curation is off')
def _get_mytardis_settings(local_settings, bdp_username): mytardis_url = local_settings['mytardis_platform'] return manage.get_platform_settings(mytardis_url, bdp_username)
def _get_mytardis_settings(local_settings, bdp_username): mytardis_url = local_settings['mytardis_platform'] return manage.get_platform_settings(mytardis_url, bdp_username)