def _get_output(self, local_settings, source_url):
"""
Retrieve the output from the task on the node
"""
logger.debug("get_output from %s" % source_url)
computation_platform_url = local_settings['comp_platform_url']
bdp_username = local_settings['bdp_username']
comp_pltf_settings = manage.get_platform_settings(
computation_platform_url, bdp_username)
local_settings.update(comp_pltf_settings)
encoded_s_url = storage.get_url_with_credentials(
local_settings,
source_url,
is_relative_path=True,
ip_address=local_settings['host'])
(scheme, host, mypath, location, query_settings) = \
storage.parse_bdpurl(encoded_s_url)
make_path = os.path.join(query_settings['root_path'], mypath)
logger.debug("make_path=%s" % make_path)
output_storage_url = local_settings['storeout_platform_url']
logger.debug("output_storage_url=%s" % output_storage_url)
output_storage_settings = manage.get_platform_settings(
output_storage_url, bdp_username)
local_settings.update(output_storage_settings)
logger.debug("output_storage_settings=%s" % output_storage_settings)
dest_url = '%s://%s@%s/%s/make%s' % (
output_storage_settings['scheme'], output_storage_settings['type'],
output_storage_settings['host'],
local_settings['storeout_platform_offset'],
str(local_settings['contextid']))
logger.debug("Transferring output from %s to %s" %
(source_url, dest_url))
local_settings.update(output_storage_settings)
encoded_d_url = storage.get_url_with_credentials(
local_settings, dest_url)
logger.debug("encoded_d_url=%s" % encoded_d_url)
# FIXME: might want to turn on paramiko compress function
#storage_files(encoded_d_url, exceptions=[])
# to speed up this transfer
try:
storage.copy_directories(encoded_s_url, encoded_d_url)
except SSHException, e:
logger.error(e)
# FIXME: Could just exit, but need to flag that this data has not
# been transferred.
raise
def _get_output(self, local_settings, source_url):
"""
Retrieve the output from the task on the node
"""
logger.debug("get_output from %s" % source_url)
computation_platform_url = local_settings['comp_platform_url']
bdp_username = local_settings['bdp_username']
comp_pltf_settings = manage.get_platform_settings(
computation_platform_url,
bdp_username)
local_settings.update(comp_pltf_settings)
encoded_s_url = storage.get_url_with_credentials(
local_settings,
source_url,
is_relative_path=True,
ip_address=local_settings['host'])
(scheme, host, mypath, location, query_settings) = \
storage.parse_bdpurl(encoded_s_url)
make_path = os.path.join(query_settings['root_path'], mypath)
logger.debug("make_path=%s" % make_path)
output_storage_url = local_settings['storeout_platform_url']
logger.debug("output_storage_url=%s" % output_storage_url)
output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username)
local_settings.update(output_storage_settings)
logger.debug("output_storage_settings=%s" % output_storage_settings)
dest_url = '%s://%s@%s/%s/make%s' % (output_storage_settings['scheme'],
output_storage_settings['type'],
output_storage_settings['host'],
local_settings['storeout_platform_offset'], str(local_settings['contextid']))
logger.debug("Transferring output from %s to %s" % (source_url,
dest_url))
local_settings.update(output_storage_settings)
encoded_d_url = storage.get_url_with_credentials(local_settings, dest_url)
logger.debug("encoded_d_url=%s" % encoded_d_url)
# FIXME: might want to turn on paramiko compress function
#storage_files(encoded_d_url, exceptions=[])
# to speed up this transfer
try:
storage.copy_directories(encoded_s_url, encoded_d_url)
except SSHException, e:
logger.error(e)
# FIXME: Could just exit, but need to flag that this data has not
# been transferred.
raise
def copy_to_scratch_space(self, run_settings, local_settings):
bdp_username = run_settings[
'http://rmit.edu.au/schemas/bdp_userprofile']['username']
output_storage_url = run_settings[
'http://rmit.edu.au/schemas/platform/storage/output'][
'platform_url']
output_storage_settings = manage.get_platform_settings(
output_storage_url, bdp_username)
run_settings['http://rmit.edu.au/schemas/platform/storage/output'][
'offset'] = self.output_loc_offset
offset = run_settings[
'http://rmit.edu.au/schemas/platform/storage/output']['offset']
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
iter_inputdir = os.path.join(self.job_dir, "input_0")
logger.debug("iter_inputdir=%s" % iter_inputdir)
input_location = run_settings[RMIT_SCHEMA +
'/input/system']['input_location']
logger.debug("input_location=%s" % input_location)
#todo: input location will evenatually be replaced by the scratch space that was used by the sweep
#todo: the sweep will indicate the location of the scratch space in the run_settings
#todo: add scheme (ssh) to inputlocation
source_url = get_url_with_credentials(local_settings, input_location)
logger.debug("source_url=%s" % source_url)
destination_url = get_url_with_credentials(
output_storage_settings,
'%s://%s@%s' % (output_storage_settings['scheme'],
output_storage_settings['type'], iter_inputdir),
is_relative_path=False)
logger.debug("destination_url=%s" % destination_url)
storage.copy_directories(source_url, destination_url)
def post_mytardis_exp(run_settings,
experiment_id,
output_location,
experiment_paramset=[]):
# TODO: move into mytardis package?
bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA)
try:
mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA)
except SettingNotFoundException:
logger.error("mytardis_platform not set")
return 0
mytardis_settings = manage.get_platform_settings(
mytardis_url,
bdp_username)
logger.debug(mytardis_settings)
curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA)
if curate_data:
if mytardis_settings['mytardis_host']:
def _get_exp_name_for_input(path):
return str(os.sep.join(path.split(os.sep)[-1:]))
ename = _get_exp_name_for_input(output_location)
logger.debug("ename=%s" % ename)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
experiment_paramset=experiment_paramset,
expname=ename)
else:
logger.warn("no mytardis host specified")
else:
logger.warn('Data curation is off')
return experiment_id
def copy_to_scratch_space(self, run_settings, local_settings):
bdp_username = run_settings['http://rmit.edu.au/schemas/bdp_userprofile']['username']
output_storage_url = run_settings['http://rmit.edu.au/schemas/platform/storage/output']['platform_url']
output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username)
run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset'] = self.output_loc_offset
offset = run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset']
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
iter_inputdir = os.path.join(self.job_dir, "input_0")
logger.debug("iter_inputdir=%s" % iter_inputdir)
input_location = run_settings[
RMIT_SCHEMA + '/input/system']['input_location']
logger.debug("input_location=%s" % input_location)
#todo: input location will evenatually be replaced by the scratch space that was used by the sweep
#todo: the sweep will indicate the location of the scratch space in the run_settings
#todo: add scheme (ssh) to inputlocation
source_url = get_url_with_credentials(local_settings, input_location)
logger.debug("source_url=%s" % source_url)
destination_url = get_url_with_credentials(
output_storage_settings,
'%s://%s@%s' % (output_storage_settings['scheme'],
output_storage_settings['type'],
iter_inputdir),
is_relative_path=False)
logger.debug("destination_url=%s" % destination_url)
storage.copy_directories(source_url, destination_url)
def curate_data(self, run_settings, location, experiment_id):
bdp_username = run_settings['http://rmit.edu.au/schemas/bdp_userprofile']['username']
curate_data = run_settings['http://rmit.edu.au/schemas/input/mytardis']['curate_data']
if curate_data:
mytardis_url = run_settings['http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform']
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
logger.debug(mytardis_settings)
EXP_DATASET_NAME_SPLIT = 2
def _get_exp_name_for_input(path):
return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
logger.debug("location=%s" % location)
ename = _get_exp_name_for_input(location)
logger.debug("ename=%s" % ename)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=[
mytardis.create_paramset("hrmcexp", []),
mytardis.create_graph_paramset("expgraph",
name="hrmcexp",
graph_info={"axes":["iteration", "criterion"], "legends":["criterion"], "precision":[0, 2]},
value_dict={},
value_keys=[["hrmcdset/it", "hrmcdset/crit"]])
])
else:
logger.warn('Data curation is off')
return experiment_id
def curate_data(self, run_settings, output_location, experiment_id):
'''
Creates experiment in MyTardis
'''
# Loading MyTardis credentials
bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % SCHEMA_PREFIX)
mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % SCHEMA_PREFIX)
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
def _get_experiment_name(path):
'''
Return the name for MyTardis experiment
e.g., if path='x/y/z', returns 'y/z'
'''
return str(os.sep.join(path.split(os.sep)[-2:]))
# Creates new experiment if experiment_id=0
# If experiment_id is non-zero, the experiment is updated
experiment_id = mytardis.create_experiment(
settings=mytardis_settings, # MyTardis credentials
exp_id=experiment_id,
expname=_get_experiment_name(output_location), # name of the experiment in MyTardis
# metadata associated with the experiment
# a list of parameter sets
experiment_paramset=[
# a new blank parameter set conforming to schema 'remotemake'
mytardis.create_paramset("remotemake", []),
# a graph parameter set
mytardis.create_graph_paramset("expgraph", # name of schema
name="randexp1", # unique graph name
graph_info={"axes":["x", "y"], "legends":["Random points"]}, # information about the graph
value_dict={}, # values to be used in parent graphs if appropriate
value_keys=[["randdset/x", "randdset/y"]]), # values from datasets to produce points in the graph
])
return experiment_id
def _upload_variations_inputs(settings, source_url_initial, values_map):
bdp_username = settings['bdp_username']
logger.debug("source_url_initial=%s" % source_url_initial)
encoded_s_url = storage.get_url_with_credentials(settings, source_url_initial)
logger.debug("encoded_s_url=%s" % encoded_s_url)
dest_url = _get_dest_bdp_url(settings)
computation_platform_url = settings['comp_platform_url']
bdp_username = settings['bdp_username']
comp_pltf_settings = manage.get_platform_settings(
computation_platform_url,
bdp_username)
settings.update(comp_pltf_settings)
encoded_d_url = storage.get_url_with_credentials(settings,
dest_url, is_relative_path=True, ip_address=settings['host'])
storage.copy_directories(encoded_s_url, encoded_d_url)
for content_fname, content in _instantiate_context(
source_url_initial,
settings,
values_map).items():
content_url = storage.get_url_with_credentials(
settings,
os.path.join(dest_url, content_fname),
is_relative_path=True, ip_address=settings['host'])
logger.debug("content_url=%s" % content_url)
storage.put_file(content_url, content.encode('utf-8'))
_save_values(settings, dest_url, values_map)
logger.debug("done input upload")
def create_experiment(self, run_settings, location, experiment_id):
bdp_username = run_settings['%s/bdp_userprofile' % django_settings.SCHEMA_PREFIX]['username']
curate_data = run_settings['%s/input/mytardis' % django_settings.SCHEMA_PREFIX]['curate_data']
if curate_data:
mytardis_url = run_settings['%s/input/mytardis' % django_settings.SCHEMA_PREFIX]['mytardis_platform']
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
logger.debug(mytardis_settings)
EXP_DATASET_NAME_SPLIT = 2
def _get_exp_name_for_input(path):
return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
logger.debug("location=%s" % location)
ename = _get_exp_name_for_input(location)
logger.debug("ename=%s" % ename)
experiment_paramset = []
self.load_metadata_builder(run_settings)
if self.METADATA_BUILDER:
experiment_paramset=self.METADATA_BUILDER.build_experiment_metadata(run_settings=run_settings)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=experiment_paramset)
else:
logger.warn('Data curation is off')
return experiment_id
def process(self, run_settings):
self.experiment_id = 0
local_settings = setup_settings(run_settings)
self.experiment_id = local_settings['experiment_id']
messages.info(run_settings, "1: waiting for completion")
logger.debug("settings=%s" % local_settings)
try:
self.runs_left = ast.literal_eval(getval(run_settings, '%s/stages/make/runs_left' % RMIT_SCHEMA))
except (ValueError, SettingNotFoundException):
self.runs_left = []
# if self._exists(run_settings,
# 'http://rmit.edu.au/schemas/stages/make',
# u'runs_left'):
# self.runs_left = ast.literal_eval(
# run_settings['http://rmit.edu.au/schemas/stages/make'][u'runs_left'])
# else:
# self.runs_left = []
def _get_dest_bdp_url(local_settings):
return "%s@%s" % (
"nci",
os.path.join(local_settings['payload_destination'],
str(local_settings['contextid'])))
dest_url = _get_dest_bdp_url(local_settings)
computation_platform_url = local_settings['comp_platform_url']
bdp_username = local_settings['bdp_username']
comp_pltf_settings = manage.get_platform_settings(
computation_platform_url,
bdp_username)
local_settings.update(comp_pltf_settings)
encoded_d_url = storage.get_url_with_credentials(
local_settings,
dest_url,
is_relative_path=True,
ip_address=local_settings['host'])
(scheme, host, mypath, location, query_settings) = \
storage.parse_bdpurl(encoded_d_url)
if self.runs_left:
job_finished = self._job_finished(
settings=local_settings,
remote_path=dest_url)
if not job_finished:
return
self._get_output(local_settings, dest_url)
self.runs_left -= 1
if self.runs_left <= 0:
messages.success(run_settings, "%s: finished" % (1))
logger.debug("processing finished")
def process(self, run_settings):
self.experiment_id = 0
local_settings = setup_settings(run_settings)
self.experiment_id = local_settings['experiment_id']
messages.info(run_settings, "1: waiting for completion")
logger.debug("settings=%s" % local_settings)
try:
self.runs_left = ast.literal_eval(
getval(run_settings, '%s/stages/make/runs_left' % RMIT_SCHEMA))
except (ValueError, SettingNotFoundException):
self.runs_left = []
# if self._exists(run_settings,
# 'http://rmit.edu.au/schemas/stages/make',
# u'runs_left'):
# self.runs_left = ast.literal_eval(
# run_settings['http://rmit.edu.au/schemas/stages/make'][u'runs_left'])
# else:
# self.runs_left = []
def _get_dest_bdp_url(local_settings):
return "%s@%s" % ("nci",
os.path.join(
local_settings['payload_destination'],
str(local_settings['contextid'])))
dest_url = _get_dest_bdp_url(local_settings)
computation_platform_url = local_settings['comp_platform_url']
bdp_username = local_settings['bdp_username']
comp_pltf_settings = manage.get_platform_settings(
computation_platform_url, bdp_username)
local_settings.update(comp_pltf_settings)
encoded_d_url = storage.get_url_with_credentials(
local_settings,
dest_url,
is_relative_path=True,
ip_address=local_settings['host'])
(scheme, host, mypath, location, query_settings) = \
storage.parse_bdpurl(encoded_d_url)
if self.runs_left:
job_finished = self._job_finished(settings=local_settings,
remote_path=dest_url)
if not job_finished:
return
self._get_output(local_settings, dest_url)
self.runs_left -= 1
if self.runs_left <= 0:
messages.success(run_settings, "%s: finished" % (1))
logger.debug("processing finished")
def curate_data(self, run_settings, location, experiment_id):
logger.debug("vasp curate_data")
logger.debug('location=%s' % location)
bdp_username = getval(
run_settings, '%s/bdp_userprofile/username' % self.SCHEMA_PREFIX)
mytardis_url = run_settings[
'http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform']
mytardis_settings = manage.get_platform_settings(
mytardis_url, bdp_username)
logger.debug(mytardis_settings)
def _get_exp_name_for_input(path):
return str(os.sep.join(path.split(os.sep)[-2:]))
ename = _get_exp_name_for_input(location)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=[
mytardis.create_paramset("remotemake", []),
mytardis.create_graph_paramset(
"expgraph",
name="makeexp1",
graph_info={
"axes": ["num_kp", "energy"],
"legends": ["TOTEN"]
},
value_dict={},
value_keys=[["makedset/num_kp", "makedset/toten"]]),
mytardis.create_graph_paramset(
"expgraph",
name="makeexp2",
graph_info={
"axes": ["encut", "energy"],
"legends": ["TOTEN"]
},
value_dict={},
value_keys=[["makedset/encut", "makedset/toten"]]),
mytardis.create_graph_paramset(
"expgraph",
name="makeexp3",
graph_info={
"axes": ["num_kp", "encut", "TOTEN"],
"legends": ["TOTEN"]
},
value_dict={},
value_keys=[[
"makedset/num_kp", "makedset/encut", "makedset/toten"
]]),
])
return experiment_id
def curate_data(self, run_settings, location, experiment_id):
logger.debug("vasp durate_data")
try:
subdirective = getval(run_settings, '%s/stages/sweep/directive' % SCHEMA_PREFIX)
except SettingNotFoundException:
logger.warn("cannot find subdirective name")
subdirective = ''
if subdirective == "vasp":
bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % SCHEMA_PREFIX)
mytardis_url = run_settings['http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform']
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
logger.debug(mytardis_settings)
def _get_exp_name_for_input(path):
return str(os.sep.join(path.split(os.sep)[-2:]))
ename = _get_exp_name_for_input(location)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=[
mytardis.create_paramset("remotemake", []),
mytardis.create_graph_paramset("expgraph",
name="makeexp1",
graph_info={"axes":["num_kp", "energy"], "legends":["TOTEN"]},
value_dict={},
value_keys=[["makedset/num_kp", "makedset/toten"]]),
mytardis.create_graph_paramset("expgraph",
name="makeexp2",
graph_info={"axes":["encut", "energy"], "legends":["TOTEN"]},
value_dict={},
value_keys=[["makedset/encut", "makedset/toten"]]),
mytardis.create_graph_paramset("expgraph",
name="makeexp3",
graph_info={"axes":["num_kp", "encut", "TOTEN"], "legends":["TOTEN"]},
value_dict={},
value_keys=[["makedset/num_kp", "makedset/encut", "makedset/toten"]]),
])
else:
logger.warn("cannot find subdirective name")
return experiment_id
def curate_data(self, run_settings, output_location, experiment_id):
'''
Creates experiment in MyTardis
'''
# Loading MyTardis credentials
bdp_username = getval(run_settings,
'%s/bdp_userprofile/username' % SCHEMA_PREFIX)
mytardis_url = getval(
run_settings,
'%s/input/mytardis/mytardis_platform' % SCHEMA_PREFIX)
mytardis_settings = manage.get_platform_settings(
mytardis_url, bdp_username)
def _get_experiment_name(path):
'''
Return the name for MyTardis experiment
e.g., if path='x/y/z', returns 'y/z'
'''
return str(os.sep.join(path.split(os.sep)[-2:]))
# Creates new experiment if experiment_id=0
# If experiment_id is non-zero, the experiment is updated
experiment_id = mytardis.create_experiment(
settings=mytardis_settings, # MyTardis credentials
exp_id=experiment_id,
expname=_get_experiment_name(
output_location), # name of the experiment in MyTardis
# metadata associated with the experiment
# a list of parameter sets
experiment_paramset=[
# a new blank parameter set conforming to schema 'remotemake'
mytardis.create_paramset("remotemake", []),
# a graph parameter set
mytardis.create_graph_paramset(
"expgraph", # name of schema
name="randexp1", # unique graph name
graph_info={
"axes": ["x", "y"],
"legends": ["Random points"]
}, # information about the graph
value_dict=
{}, # values to be used in parent graphs if appropriate
value_keys=[[
"randdset/x", "randdset/y"
]]), # values from datasets to produce points in the graph
])
return experiment_id
def copy_to_scratch_space(self, run_settings, local_settings, result_offset):
bdp_username = run_settings['%s/bdp_userprofile' % django_settings.SCHEMA_PREFIX]['username']
output_storage_url = run_settings['%s/platform/storage/output' % django_settings.SCHEMA_PREFIX]['platform_url']
output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username)
run_settings['%s/platform/storage/output' % django_settings.SCHEMA_PREFIX]['offset'] = self.output_loc_offset
offset = run_settings['%s/platform/storage/output' % django_settings.SCHEMA_PREFIX]['offset']
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
iter_inputdir = os.path.join(self.job_dir, result_offset)
logger.debug("iter_inputdir=%s" % iter_inputdir)
input_storage_settings = self.get_platform_settings(run_settings, '%s/platform/storage/input' % django_settings.SCHEMA_PREFIX)
#input_location = run_settings[django_settings.SCHEMA_PREFIX + '/input/system']['input_location']
try:
input_location = getval(run_settings, django_settings.SCHEMA_PREFIX + '/input/system/input_location')
except SettingNotFoundException:
try:
input_location = getval(run_settings, django_settings.SCHEMA_PREFIX + '/input/location/input_location')
except:
input_location = getval(run_settings, django_settings.SCHEMA_PREFIX + '/input/location/input/input_location')
logger.debug("input_location=%s" % input_location)
#todo: input location will evenatually be replaced by the scratch space that was used by the sweep
#todo: the sweep will indicate the location of the scratch space in the run_settings
#todo: add scheme (ssh) to inputlocation
#source_url = get_url_with_credentials(local_settings, input_location)
input_offset = run_settings['%s/platform/storage/input' % django_settings.SCHEMA_PREFIX]['offset']
input_url = "%s://%s@%s/%s" % (input_storage_settings['scheme'],
input_storage_settings['type'],
input_storage_settings['host'], input_offset)
source_url = get_url_with_credentials(
input_storage_settings, input_url, is_relative_path=False)
logger.debug("source_url=%s" % source_url)
destination_url = get_url_with_credentials(
output_storage_settings,
'%s://%s@%s' % (output_storage_settings['scheme'],
output_storage_settings['type'],
iter_inputdir),
is_relative_path=False)
logger.debug("destination_url=%s" % destination_url)
storage.copy_directories(source_url, destination_url)
def curate_data(self, run_settings, location, experiment_id):
bdp_username = run_settings[
'http://rmit.edu.au/schemas/bdp_userprofile']['username']
curate_data = run_settings[
'http://rmit.edu.au/schemas/input/mytardis']['curate_data']
if curate_data:
mytardis_url = run_settings[
'http://rmit.edu.au/schemas/input/mytardis'][
'mytardis_platform']
mytardis_settings = manage.get_platform_settings(
mytardis_url, bdp_username)
logger.debug(mytardis_settings)
EXP_DATASET_NAME_SPLIT = 2
def _get_exp_name_for_input(path):
return str(
os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
logger.debug("location=%s" % location)
ename = _get_exp_name_for_input(location)
logger.debug("ename=%s" % ename)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=[
mytardis.create_paramset("hrmcexp", []),
mytardis.create_graph_paramset(
"expgraph",
name="hrmcexp",
graph_info={
"axes": ["iteration", "criterion"],
"legends": ["criterion"],
"precision": [0, 2]
},
value_dict={},
value_keys=[["hrmcdset/it", "hrmcdset/crit"]])
])
else:
logger.warn('Data curation is off')
return experiment_id
def process(self, run_settings):
""" perform the stage operation
"""
bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA)
# bdp_username = run_settings[RMIT_SCHEMA + '/bdp_userprofile']['username']
logger.debug("bdp_username=%s" % bdp_username)
input_storage_url = getval(run_settings,
'%s/platform/storage/input/platform_url'
% RMIT_SCHEMA)
# input_storage_url = run_settings[
# RMIT_SCHEMA + '/platform/storage/input']['platform_url']
logger.debug("input_storage_url=%s" % input_storage_url)
input_storage_settings = manage.get_platform_settings(
input_storage_url,
bdp_username)
logger.debug("input_storage_settings=%s" % pformat(input_storage_settings))
input_offset = getval(run_settings,
'%s/platform/storage/input/offset'
% RMIT_SCHEMA)
# input_offset = run_settings[RMIT_SCHEMA + "/platform/storage/input"]['offset']
logger.debug("input_offset=%s" % pformat(input_offset))
input_prefix = '%s://%s@' % (input_storage_settings['scheme'],
input_storage_settings['type'])
map_initial_location = "%s/%s/initial" % (input_prefix, input_offset)
logger.debug("map_initial_location=%s" % map_initial_location)
local_settings = setup_settings(run_settings)
logger.debug("local_settings=%s" % local_settings)
values_map = _load_values_map(local_settings, map_initial_location)
logger.debug("values_map=%s" % values_map)
_upload_variations_inputs(
local_settings,
map_initial_location, values_map)
_upload_payload(local_settings, local_settings['payload_source'], values_map)
messages.info(run_settings, "1: upload done")
def create_experiment(self, run_settings, location, experiment_id):
bdp_username = run_settings['%s/bdp_userprofile' %
django_settings.SCHEMA_PREFIX]['username']
curate_data = run_settings[
'%s/input/mytardis' % django_settings.SCHEMA_PREFIX]['curate_data']
if curate_data:
mytardis_url = run_settings[
'%s/input/mytardis' %
django_settings.SCHEMA_PREFIX]['mytardis_platform']
mytardis_settings = manage.get_platform_settings(
mytardis_url, bdp_username)
logger.debug(mytardis_settings)
EXP_DATASET_NAME_SPLIT = 2
def _get_exp_name_for_input(path):
return str(
os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
logger.debug("location=%s" % location)
ename = _get_exp_name_for_input(location)
logger.debug("ename=%s" % ename)
experiment_paramset = []
self.load_metadata_builder(run_settings)
if self.METADATA_BUILDER:
experiment_paramset = self.METADATA_BUILDER.build_experiment_metadata(
run_settings=run_settings)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=experiment_paramset)
else:
logger.warn('Data curation is off')
return experiment_id
def curate_data(self, run_settings, location, experiment_id):
logger.debug("vasp curate_data")
logger.debug('location=%s' % location)
bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % self.SCHEMA_PREFIX)
mytardis_url = run_settings['http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform']
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
logger.debug(mytardis_settings)
def _get_exp_name_for_input(path):
return str(os.sep.join(path.split(os.sep)[-2:]))
ename = _get_exp_name_for_input(location)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=[
mytardis.create_paramset("remotemake", []),
mytardis.create_graph_paramset("expgraph",
name="makeexp1",
graph_info={"axes":["num_kp", "energy"], "legends":["TOTEN"]},
value_dict={},
value_keys=[["makedset/num_kp", "makedset/toten"]]),
mytardis.create_graph_paramset("expgraph",
name="makeexp2",
graph_info={"axes":["encut", "energy"], "legends":["TOTEN"]},
value_dict={},
value_keys=[["makedset/encut", "makedset/toten"]]),
mytardis.create_graph_paramset("expgraph",
name="makeexp3",
graph_info={"axes":["num_kp", "encut", "TOTEN"], "legends":["TOTEN"]},
value_dict={},
value_keys=[["makedset/num_kp", "makedset/encut", "makedset/toten"]]),
])
return experiment_id
def process(self, run_settings):
settings = setup_settings(run_settings)
messages.info(run_settings, "1: execute starting")
def _get_dest_bdp_url(settings):
return "%s@%s" % (
"nci",
os.path.join(settings['payload_destination'],
str(settings['contextid'])))
dest_url = _get_dest_bdp_url(settings)
computation_platform_url = settings['comp_platform_url']
bdp_username = settings['bdp_username']
comp_pltf_settings = manage.get_platform_settings(
computation_platform_url,
bdp_username)
logger.debug("comp_pltf_settings=%s" % pformat(comp_pltf_settings))
settings.update(comp_pltf_settings)
encoded_d_url = storage.get_url_with_credentials(
settings,
dest_url,
is_relative_path=True,
ip_address=settings['host'])
(scheme, host, mypath, location, query_settings) = \
storage.parse_bdpurl(encoded_d_url)
stderr = ''
try:
ssh = open_connection(
ip_address=settings['host'],
settings=settings)
(command_out, stderr) = compute.run_make(ssh, (os.path.join(
query_settings['root_path'],
mypath)), 'startrun')
except Exception, e:
logger.error(e)
raise
def process(self, run_settings):
# self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)
bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA)
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA)
output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username)
logger.debug("output_storage_settings=%s" % output_storage_settings)
output_prefix = '%s://%s@' % (output_storage_settings['scheme'],
output_storage_settings['type'])
offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA)
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
self.output_dir = os.path.join(os.path.join(self.job_dir, "output_%s" % self.id))
self.input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % self.id))
self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % (self.id + 1)))
except (SettingNotFoundException, ValueError):
# FIXME: Not clear that this a valid path through stages
self.output_dir = os.path.join(os.path.join(self.job_dir, "output"))
self.output_dir = os.path.join(os.path.join(self.job_dir, "input"))
self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_1"))
logger.debug('self.output_dir=%s' % self.output_dir)
try:
self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA))
except SettingNotFoundException:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
output_url = get_url_with_credentials(
output_storage_settings,
output_prefix + self.output_dir, is_relative_path=False)
# (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
# fsys = storage.get_filesystem(output_url)
# node_output_dirs, _ = fsys.listdir(mypath)
# logger.debug("node_output_dirs=%s" % node_output_dirs)
self.process_outputs(run_settings, self.job_dir, output_url, output_storage_settings, offset)
# logger.debug("output_url=%s" % output_url)
# # Should this be output_dir or root of remotesys?
# (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
# fsys = storage.get_filesystem(output_url)
# logger.debug("fsys=%s" % fsys)
# logger.debug("mypath=%s" % mypath)
# node_output_dirs, _ = fsys.listdir(mypath)
# logger.debug("node_output_dirs=%s" % node_output_dirs)
# self.audit = ""
# outputs = []
# Node_info = namedtuple('Node_info',
# ['dir', 'index', 'number', 'criterion'])
# # gather node_infos
# for node_output_dir in node_output_dirs:
# base_fname = "HRMC.inp"
# try:
# values_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.output_dir, node_output_dir,
# '%s_values' % base_fname), is_relative_path=False)
# values_content = storage.get_file(values_url)
# logger.debug("values_file=%s" % values_url)
# except IOError:
# logger.warn("no values file found")
# values_map = {}
# else:
# values_map = dict(json.loads(values_content))
# criterion = self.compute_psd_criterion(
# node_output_dir, fsys,
# output_storage_settings)
# #criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dir, fs,)
# logger.debug("criterion=%s" % criterion)
# index = 0 # FIXME: as node_output_dirs in particular order, then index is not useful.
# outputs.append(Node_info(dir=node_output_dir,
# index=index, number=values_map['run_counter'], criterion=criterion))
# outputs.sort(key=lambda x: int(x.criterion))
# logger.debug("outputs=%s" % outputs)
# logger.debug('threshold=%s' % self.threshold)
# total_picks = 1
# if len(self.threshold) > 1:
# for i in self.threshold:
# total_picks *= self.threshold[i]
# else:
# total_picks = self.threshold[0]
# if not outputs:
# logger.error("no ouput found for this iteration")
# return
# for index in range(0, total_picks):
# Node_info = outputs[index]
# logger.debug("node_info.dir=%s" % Node_info.dir)
# logger.debug("Node_info=%s" % str(Node_info))
# self.new_input_node_dir = os.path.join(self.new_input_dir,
# Node_info.dir)
# logger.debug("New input node dir %s" % self.new_input_node_dir)
# # Move all existing domain input files unchanged to next input directory
# for f in self.DOMAIN_INPUT_FILES:
# source_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.output_dir, Node_info.dir, f), is_relative_path=False)
# dest_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_node_dir, f),
# is_relative_path=False)
# logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url))
# content = storage.get_file(source_url)
# logger.debug('content collected')
# storage.put_file(dest_url, content)
# logger.debug('put successfully')
# logger.debug('put file successfully')
# pattern = "*_values"
# self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir),
# self.new_input_node_dir, pattern,
# output_storage_settings)
# pattern = "*_template"
# self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir),
# self.new_input_node_dir, pattern,
# output_storage_settings)
# # NB: Converge stage triggers based on criterion value from audit.
# info = "Run %s preserved (error %s)\n" % (Node_info.number, Node_info.criterion)
# audit_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_node_dir, 'audit.txt'), is_relative_path=False)
# storage.put_file(audit_url, info)
# logger.debug("audit=%s" % info)
# self.audit += info
# # move xyz_final.xyz to initial.xyz
# source_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.output_dir, Node_info.dir, "xyz_final.xyz"), is_relative_path=False)
# dest_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_node_dir, 'input_initial.xyz'), is_relative_path=False)
# content = storage.get_file(source_url)
# storage.put_file(dest_url, content)
# self.audit += "spawning diamond runs\n"
# audit_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_dir, 'audit.txt'), is_relative_path=False)
# storage.put_file(audit_url, self.audit)
# curate dataset into mytardis
try:
curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA)
except SettingNotFoundException:
curate_data = 0
if curate_data:
mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA)
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
all_settings = dict(mytardis_settings)
all_settings.update(output_storage_settings)
self.experiment_id = self.curate_dataset(run_settings, self.experiment_id,
self.job_dir, output_url,
all_settings)
else:
logger.warn('Data curation is off')
def process(self, run_settings):
try:
id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
id = 0
messages.info(run_settings, '%d: transforming' % (id+1))
# self.contextid = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX)
bdp_username = getval(run_settings, '%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX)
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX)
output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username)
logger.debug("output_storage_settings=%s" % output_storage_settings)
output_prefix = '%s://%s@' % (output_storage_settings['scheme'],
output_storage_settings['type'])
offset = getval(run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX)
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
try:
self.id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX))
self.output_dir = os.path.join(os.path.join(self.job_dir, "output_%s" % self.id))
self.input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % self.id))
self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_%d" % (self.id + 1)))
except (SettingNotFoundException, ValueError):
# FIXME: Not clear that this a valid path through stages
self.output_dir = os.path.join(os.path.join(self.job_dir, "output"))
self.output_dir = os.path.join(os.path.join(self.job_dir, "input"))
self.new_input_dir = os.path.join(os.path.join(self.job_dir, "input_1"))
logger.debug('self.output_dir=%s' % self.output_dir)
try:
self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX))
except SettingNotFoundException:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
output_url = get_url_with_credentials(
output_storage_settings,
output_prefix + self.output_dir, is_relative_path=False)
# (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
# fsys = storage.get_filesystem(output_url)
# node_output_dirs, _ = fsys.listdir(mypath)
# logger.debug("node_output_dirs=%s" % node_output_dirs)
outputs = self.process_outputs(run_settings, self.job_dir, output_url, output_storage_settings, offset)
try:
curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
curate_data = 0
if curate_data:
mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % django_settings.SCHEMA_PREFIX)
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
all_settings = dict(mytardis_settings)
all_settings.update(output_storage_settings)
all_settings['contextid'] = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX)
try:
mytardis_platform = jobs.safe_import('chiminey.platform.mytardis.MyTardisPlatform', [], {})
logger.debug('self_outpus=%s' % outputs)
self.experiment_id = mytardis_platform.create_dataset_for_intermediate_output(run_settings, self.experiment_id, self.job_dir, output_url, all_settings, outputs=outputs)
except ImproperlyConfigured as e:
logger.error("Cannot load mytardis platform hook %s" % e)
else:
logger.warn('Data curation is off')
def process(self, run_settings):
try:
id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
id = 0
messages.info(run_settings, '%d: converging' % (id+1))
def retrieve_local_settings(run_settings, local_settings):
update(local_settings, run_settings
# '%s/stages/setup/payload_source' % RMIT_SCHEMA,
# '%s/stages/setup/payload_destination' % RMIT_SCHEMA,
# '%s/system/platform' % RMIT_SCHEMA,
# # '%s/stages/create/custom_prompt' % RMIT_SCHEMA,
# # '%s/stages/create/cloud_sleep_interval' % RMIT_SCHEMA,
# # '%s/stages/create/created_nodes' % RMIT_SCHEMA,
# '%s/stages/run/payload_cloud_dirname' % RMIT_SCHEMA,
# '%s/system/max_seed_int' % RMIT_SCHEMA,
# '%s/stages/run/compile_file' % RMIT_SCHEMA,
# '%s/stages/run/retry_attempts' % RMIT_SCHEMA,
# '%s/input/system/cloud/number_vm_instances' % RMIT_SCHEMA,
# '%s/input/hrmc/iseed' % RMIT_SCHEMA,
# '%s/input/hrmc/optimisation_scheme' % RMIT_SCHEMA,
# '%s/input/hrmc/threshold' % RMIT_SCHEMA,
)
local_settings['bdp_username'] = getval(run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA)
local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)
retrieve_local_settings(run_settings, local_settings)
bdp_username = local_settings['bdp_username']
# get output
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA)
output_storage_settings = manage.get_platform_settings(output_storage_url, bdp_username)
output_prefix = '%s://%s@' % (output_storage_settings['scheme'],
output_storage_settings['type'])
offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA)
job_dir = manage.get_job_dir(output_storage_settings, offset)
# get mytardis
#mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA)
#mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
# setup new paths
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
self.output_dir = os.path.join(job_dir, "output_%d" % self.id)
self.iter_inputdir = os.path.join(job_dir, "input_%d" % (self.id + 1))
#self.new_iter_inputdir = "input_%d" % (self.id + 1)
except (SettingNotFoundException, ValueError):
self.output_dir = os.path.join(job_dir, "output")
self.iter_inputdir = os.path.join(job_dir, "input")
self.id = 0
logger.debug('output_dir=%s iter_inputdir=%s' % (self.output_dir, self.iter_inputdir))
try:
self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA))
except SettingNotFoundException:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
inputdir_url = get_url_with_credentials(output_storage_settings,
output_prefix + self.iter_inputdir, is_relative_path=False)
logger.debug('input_dir_url=%s' % inputdir_url)
# (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(inputdir_url)
# fsys = storage.get_filesystem(inputdir_url)
# logger.debug('mypath=%s' % mypath)
# input_dirs, _ = fsys.listdir(mypath)
# logger.debug('input_dirs=%s' % input_dirs)
(self.done_iterating, self.criterion) = self.process_outputs(run_settings, job_dir, inputdir_url, output_storage_settings)
if self.done_iterating:
logger.debug("Total Iterations: %d" % self.id)
# output_prefix = '%s://%s@' % (output_storage_settings['scheme'],
# output_storage_settings['type'])
# new_output_dir = os.path.join(base_dir, 'output')
output_prefix = '%s://%s@' % (output_storage_settings['scheme'],
output_storage_settings['type'])
# get source url
iter_output_dir = os.path.join(os.path.join(job_dir, "output_%s" % self.id))
source_url = "%s%s" % (output_prefix, iter_output_dir)
# get dest url
new_output_dir = os.path.join(job_dir, 'output')
dest_url = "%s%s" % (output_prefix, new_output_dir)
source_url = get_url_with_credentials(output_storage_settings,
output_prefix + os.path.join(iter_output_dir), is_relative_path=False)
dest_url = get_url_with_credentials(output_storage_settings,
output_prefix + os.path.join(new_output_dir), is_relative_path=False)
storage.copy_directories(source_url, dest_url)
# curate
try:
curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA)
except SettingNotFoundException:
curate_data = 0
if curate_data:
mytardis_url = getval(run_settings, '%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA)
mytardis_settings = manage.get_platform_settings(mytardis_url, bdp_username)
all_settings = dict(mytardis_settings)
all_settings.update(output_storage_settings)
logger.debug("source_url=%s" % source_url)
logger.debug("dest_url=%s" % dest_url)
logger.debug("job_dir=%s" % job_dir)
self.experiment_id = self.curate_dataset(run_settings, self.experiment_id,
job_dir, dest_url,
all_settings)
else:
logger.warn('Data curation is off')
self.rand_index = 42
logger.debug(e)
logger.debug("processing execute stage")
local_settings = getvals(
run_settings, models.UserProfile.PROFILE_SCHEMA_NS)
self.set_execute_settings(run_settings, local_settings)
self.contextid = getval(
run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX)
# NB: Don't catch SettingNotFoundException because we can't recover
# run_settings['%s/system' % django_settings.SCHEMA_PREFIX][u'contextid']
logger.debug('contextid=%s' % self.contextid)
output_storage_url = getval(
run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX)
output_storage_settings = manage.get_platform_settings(
output_storage_url, local_settings['bdp_username'])
offset = getval(
run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX)
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
# TODO: we assume initial input is in "%s/input_0" % self.job_dir
# in configure stage we could copy initial data in 'input_location'
# into this location
try:
self.id = int(getval(run_settings, '%s/system/id' %
django_settings.SCHEMA_PREFIX))
self.iter_inputdir = os.path.join(
self.job_dir, "input_%s" % self.id)
except (SettingNotFoundException, ValueError):
self.id = 0
self.iter_inputdir = os.path.join(self.job_dir, "input_location")
messages.info(run_settings, "%s: Executing" % (self.id + 1))
except SettingNotFoundException, e:
self.rand_index = 42
logger.debug(e)
logger.debug("processing execute stage")
local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)
#self.retrieve_boto_settings(run_settings, local_settings)
self.set_execute_settings(run_settings, local_settings)
self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)
# NB: Don't catch SettingNotFoundException because we can't recover
# run_settings['http://rmit.edu.au/schemas/system'][u'contextid']
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA)
output_storage_settings = manage.get_platform_settings(output_storage_url, local_settings['bdp_username'])
offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA)
# offset = run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset']
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
# TODO: we assume initial input is in "%s/input_0" % self.job_dir
# in configure stage we could copy initial data in 'input_location' into this location
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
self.iter_inputdir = os.path.join(self.job_dir, "input_%s" % self.id)
except (SettingNotFoundException, ValueError):
self.id = 0
self.iter_inputdir = os.path.join(self.job_dir, "input_location")
messages.info(run_settings, "%s: execute" % (self.id + 1))
logger.debug("id = %s" % self.id)
try:
def curate_data(self, run_settings, location, experiment_id):
logger.debug("vasp durate_data")
try:
subdirective = getval(run_settings,
'%s/stages/sweep/directive' % SCHEMA_PREFIX)
except SettingNotFoundException:
logger.warn("cannot find subdirective name")
subdirective = ''
if subdirective == "vasp":
bdp_username = getval(
run_settings, '%s/bdp_userprofile/username' % SCHEMA_PREFIX)
mytardis_url = run_settings[
'http://rmit.edu.au/schemas/input/mytardis'][
'mytardis_platform']
mytardis_settings = manage.get_platform_settings(
mytardis_url, bdp_username)
logger.debug(mytardis_settings)
def _get_exp_name_for_input(path):
return str(os.sep.join(path.split(os.sep)[-2:]))
ename = _get_exp_name_for_input(location)
experiment_id = mytardis.create_experiment(
settings=mytardis_settings,
exp_id=experiment_id,
expname=ename,
experiment_paramset=[
mytardis.create_paramset("remotemake", []),
mytardis.create_graph_paramset(
"expgraph",
name="makeexp1",
graph_info={
"axes": ["num_kp", "energy"],
"legends": ["TOTEN"]
},
value_dict={},
value_keys=[["makedset/num_kp", "makedset/toten"]]),
mytardis.create_graph_paramset(
"expgraph",
name="makeexp2",
graph_info={
"axes": ["encut", "energy"],
"legends": ["TOTEN"]
},
value_dict={},
value_keys=[["makedset/encut", "makedset/toten"]]),
mytardis.create_graph_paramset(
"expgraph",
name="makeexp3",
graph_info={
"axes": ["num_kp", "encut", "TOTEN"],
"legends": ["TOTEN"]
},
value_dict={},
value_keys=[[
"makedset/num_kp", "makedset/encut",
"makedset/toten"
]]),
])
else:
logger.warn("cannot find subdirective name")
return experiment_id
def process(self, run_settings):
# self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)
bdp_username = getval(run_settings,
'%s/bdp_userprofile/username' % RMIT_SCHEMA)
output_storage_url = getval(
run_settings,
'%s/platform/storage/output/platform_url' % RMIT_SCHEMA)
output_storage_settings = manage.get_platform_settings(
output_storage_url, bdp_username)
logger.debug("output_storage_settings=%s" % output_storage_settings)
output_prefix = '%s://%s@' % (output_storage_settings['scheme'],
output_storage_settings['type'])
offset = getval(run_settings,
'%s/platform/storage/output/offset' % RMIT_SCHEMA)
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
self.output_dir = os.path.join(
os.path.join(self.job_dir, "output_%s" % self.id))
self.input_dir = os.path.join(
os.path.join(self.job_dir, "input_%d" % self.id))
self.new_input_dir = os.path.join(
os.path.join(self.job_dir, "input_%d" % (self.id + 1)))
except (SettingNotFoundException, ValueError):
# FIXME: Not clear that this a valid path through stages
self.output_dir = os.path.join(os.path.join(
self.job_dir, "output"))
self.output_dir = os.path.join(os.path.join(self.job_dir, "input"))
self.new_input_dir = os.path.join(
os.path.join(self.job_dir, "input_1"))
logger.debug('self.output_dir=%s' % self.output_dir)
try:
self.experiment_id = int(
getval(run_settings,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA))
except SettingNotFoundException:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
output_url = get_url_with_credentials(output_storage_settings,
output_prefix + self.output_dir,
is_relative_path=False)
# (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
# fsys = storage.get_filesystem(output_url)
# node_output_dirs, _ = fsys.listdir(mypath)
# logger.debug("node_output_dirs=%s" % node_output_dirs)
self.process_outputs(run_settings, self.job_dir, output_url,
output_storage_settings, offset)
# logger.debug("output_url=%s" % output_url)
# # Should this be output_dir or root of remotesys?
# (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
# fsys = storage.get_filesystem(output_url)
# logger.debug("fsys=%s" % fsys)
# logger.debug("mypath=%s" % mypath)
# node_output_dirs, _ = fsys.listdir(mypath)
# logger.debug("node_output_dirs=%s" % node_output_dirs)
# self.audit = ""
# outputs = []
# Node_info = namedtuple('Node_info',
# ['dir', 'index', 'number', 'criterion'])
# # gather node_infos
# for node_output_dir in node_output_dirs:
# base_fname = "HRMC.inp"
# try:
# values_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.output_dir, node_output_dir,
# '%s_values' % base_fname), is_relative_path=False)
# values_content = storage.get_file(values_url)
# logger.debug("values_file=%s" % values_url)
# except IOError:
# logger.warn("no values file found")
# values_map = {}
# else:
# values_map = dict(json.loads(values_content))
# criterion = self.compute_psd_criterion(
# node_output_dir, fsys,
# output_storage_settings)
# #criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dir, fs,)
# logger.debug("criterion=%s" % criterion)
# index = 0 # FIXME: as node_output_dirs in particular order, then index is not useful.
# outputs.append(Node_info(dir=node_output_dir,
# index=index, number=values_map['run_counter'], criterion=criterion))
# outputs.sort(key=lambda x: int(x.criterion))
# logger.debug("outputs=%s" % outputs)
# logger.debug('threshold=%s' % self.threshold)
# total_picks = 1
# if len(self.threshold) > 1:
# for i in self.threshold:
# total_picks *= self.threshold[i]
# else:
# total_picks = self.threshold[0]
# if not outputs:
# logger.error("no ouput found for this iteration")
# return
# for index in range(0, total_picks):
# Node_info = outputs[index]
# logger.debug("node_info.dir=%s" % Node_info.dir)
# logger.debug("Node_info=%s" % str(Node_info))
# self.new_input_node_dir = os.path.join(self.new_input_dir,
# Node_info.dir)
# logger.debug("New input node dir %s" % self.new_input_node_dir)
# # Move all existing domain input files unchanged to next input directory
# for f in self.DOMAIN_INPUT_FILES:
# source_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.output_dir, Node_info.dir, f), is_relative_path=False)
# dest_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_node_dir, f),
# is_relative_path=False)
# logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url))
# content = storage.get_file(source_url)
# logger.debug('content collected')
# storage.put_file(dest_url, content)
# logger.debug('put successfully')
# logger.debug('put file successfully')
# pattern = "*_values"
# self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir),
# self.new_input_node_dir, pattern,
# output_storage_settings)
# pattern = "*_template"
# self.copy_files_with_pattern(fsys, os.path.join(self.output_dir, Node_info.dir),
# self.new_input_node_dir, pattern,
# output_storage_settings)
# # NB: Converge stage triggers based on criterion value from audit.
# info = "Run %s preserved (error %s)\n" % (Node_info.number, Node_info.criterion)
# audit_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_node_dir, 'audit.txt'), is_relative_path=False)
# storage.put_file(audit_url, info)
# logger.debug("audit=%s" % info)
# self.audit += info
# # move xyz_final.xyz to initial.xyz
# source_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.output_dir, Node_info.dir, "xyz_final.xyz"), is_relative_path=False)
# dest_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_node_dir, 'input_initial.xyz'), is_relative_path=False)
# content = storage.get_file(source_url)
# storage.put_file(dest_url, content)
# self.audit += "spawning diamond runs\n"
# audit_url = get_url_with_credentials(
# output_storage_settings,
# output_prefix + os.path.join(self.new_input_dir, 'audit.txt'), is_relative_path=False)
# storage.put_file(audit_url, self.audit)
# curate dataset into mytardis
try:
curate_data = getval(run_settings,
'%s/input/mytardis/curate_data' % RMIT_SCHEMA)
except SettingNotFoundException:
curate_data = 0
if curate_data:
mytardis_url = getval(
run_settings,
'%s/input/mytardis/mytardis_platform' % RMIT_SCHEMA)
mytardis_settings = manage.get_platform_settings(
mytardis_url, bdp_username)
all_settings = dict(mytardis_settings)
all_settings.update(output_storage_settings)
self.experiment_id = self.curate_dataset(run_settings,
self.experiment_id,
self.job_dir, output_url,
all_settings)
else:
logger.warn('Data curation is off')
def process(self, run_settings):
try:
id = int(
getval(run_settings,
'%s/system/id' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
id = 0
messages.info(run_settings, '%d: transforming' % (id + 1))
# self.contextid = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX)
bdp_username = getval(
run_settings,
'%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX)
output_storage_url = getval(
run_settings, '%s/platform/storage/output/platform_url' %
django_settings.SCHEMA_PREFIX)
output_storage_settings = manage.get_platform_settings(
output_storage_url, bdp_username)
logger.debug("output_storage_settings=%s" % output_storage_settings)
output_prefix = '%s://%s@' % (output_storage_settings['scheme'],
output_storage_settings['type'])
offset = getval(
run_settings, '%s/platform/storage/output/offset' %
django_settings.SCHEMA_PREFIX)
self.job_dir = manage.get_job_dir(output_storage_settings, offset)
try:
self.id = int(
getval(run_settings,
'%s/system/id' % django_settings.SCHEMA_PREFIX))
self.output_dir = os.path.join(
os.path.join(self.job_dir, "output_%s" % self.id))
self.input_dir = os.path.join(
os.path.join(self.job_dir, "input_%d" % self.id))
self.new_input_dir = os.path.join(
os.path.join(self.job_dir, "input_%d" % (self.id + 1)))
except (SettingNotFoundException, ValueError):
# FIXME: Not clear that this a valid path through stages
self.output_dir = os.path.join(os.path.join(
self.job_dir, "output"))
self.output_dir = os.path.join(os.path.join(self.job_dir, "input"))
self.new_input_dir = os.path.join(
os.path.join(self.job_dir, "input_1"))
logger.debug('self.output_dir=%s' % self.output_dir)
try:
self.experiment_id = int(
getval(
run_settings, '%s/input/mytardis/experiment_id' %
django_settings.SCHEMA_PREFIX))
except SettingNotFoundException:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
output_url = get_url_with_credentials(output_storage_settings,
output_prefix + self.output_dir,
is_relative_path=False)
# (scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
# fsys = storage.get_filesystem(output_url)
# node_output_dirs, _ = fsys.listdir(mypath)
# logger.debug("node_output_dirs=%s" % node_output_dirs)
outputs = self.process_outputs(run_settings, self.job_dir, output_url,
output_storage_settings, offset)
try:
curate_data = getval(
run_settings, '%s/input/mytardis/curate_data' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
curate_data = 0
if curate_data:
mytardis_url = getval(
run_settings, '%s/input/mytardis/mytardis_platform' %
django_settings.SCHEMA_PREFIX)
mytardis_settings = manage.get_platform_settings(
mytardis_url, bdp_username)
all_settings = dict(mytardis_settings)
all_settings.update(output_storage_settings)
all_settings['contextid'] = getval(
run_settings,
'%s/system/contextid' % django_settings.SCHEMA_PREFIX)
try:
mytardis_platform = jobs.safe_import(
'chiminey.platform.mytardis.MyTardisPlatform', [], {})
logger.debug('self_outpus=%s' % outputs)
self.experiment_id = mytardis_platform.create_dataset_for_intermediate_output(
run_settings,
self.experiment_id,
self.job_dir,
output_url,
all_settings,
outputs=outputs)
except ImproperlyConfigured as e:
logger.error("Cannot load mytardis platform hook %s" % e)
else:
logger.warn('Data curation is off')
def _get_mytardis_settings(local_settings, bdp_username):
mytardis_url = local_settings['mytardis_platform']
return manage.get_platform_settings(mytardis_url, bdp_username)
def _get_mytardis_settings(local_settings, bdp_username):
mytardis_url = local_settings['mytardis_platform']
return manage.get_platform_settings(mytardis_url, bdp_username)
