def create_dataset_for_final_output(self, run_settings, experiment_id, base_dir, output_url, all_settings):
logger.debug("curate_dataset")
iter_output_dir = os.path.join(os.path.join(base_dir, "output"))
logger.debug("iter_output_dir=%s" % iter_output_dir)
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
logger.debug("iter_output_dir=%s" % iter_output_dir)
logger.debug("output_url=%s" % output_url)
(scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
fsys = storage.get_filesystem(output_url)
node_output_dirnames, _ = fsys.listdir(mypath)
logger.debug("node_output_dirnames=%s" % node_output_dirnames)
curate_data = (getval(run_settings, '%s/input/mytardis/curate_data' % self.SCHEMA_PREFIX))
if curate_data:
if all_settings['mytardis_host']:
output_dirs = []
for m, dir_name in enumerate(node_output_dirnames):
output_dirs.append(os.path.join(iter_output_dir, dir_name))
for m, output_dir in enumerate(output_dirs):
#node_path = os.path.join(iter_output_dir, node_dir)
logger.debug("output_dir=%s" % output_dir)
dataset_paramset = []
datafile_paramset = []
dfile_extract_func = {}
self.load_metadata_builder(run_settings)
if self.METADATA_BUILDER:
(experiment_paramset, dataset_paramset, datafile_paramset, dfile_extract_func) = \
self.METADATA_BUILDER.build_metadata_for_final_output(m, output_dir, \
run_settings=run_settings, storage_settings=all_settings,\
output_dirs=output_dirs)
source_url = get_url_with_credentials(
all_settings, output_dir, is_relative_path=False)
logger.debug("source_url=%s" % source_url)
experiment_id = mytardis.create_dataset(
settings=all_settings,
source_url=source_url,
exp_name=mytardis.get_exp_name_for_output,
dataset_name=mytardis.get_dataset_name_for_output,
exp_id=experiment_id,
experiment_paramset=experiment_paramset,
dataset_paramset=dataset_paramset,
datafile_paramset=datafile_paramset,
dfile_extract_func=dfile_extract_func)
graph_paramset = []
else:
logger.warn("no mytardis host specified")
else:
logger.warn('Data curation is off')
return experiment_id
def create_dataset_for_intermediate_output(self, run_settings, experiment_id, base_dir, output_url,
all_settings, outputs=[]):
logger.debug('self_outpus_curate=%s' % outputs)
iteration = int(getval(run_settings, '%s/system/id' % self.SCHEMA_PREFIX))
iter_output_dir = os.path.join(os.path.join(base_dir, "output_%s" % iteration))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
(scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
fsys = storage.get_filesystem(output_url)
node_output_dirnames, _ = fsys.listdir(mypath)
logger.debug("node_output_dirnames=%s" % node_output_dirnames)
if all_settings['mytardis_host']:
output_dirs = []
for m, dir_name in enumerate(node_output_dirnames):
output_dirs.append(os.path.join(iter_output_dir, dir_name))
for i, output_dir in enumerate(output_dirs):
dataset_paramset = []
datafile_paramset = []
dfile_extract_func = {}
self.load_metadata_builder(run_settings)
if self.METADATA_BUILDER:
(continue_loop, dataset_paramset, datafile_paramset, dfile_extract_func) = \
self.METADATA_BUILDER.build_metadata_for_intermediate_output(\
output_dir, outputs, run_settings=run_settings, storage_settings=all_settings,\
output_dirs=output_dirs)
if continue_loop:
continue
source_dir_url = get_url_with_credentials(
all_settings,
output_dir,
is_relative_path=False)
logger.debug("source_dir_url=%s" % source_dir_url)
logger.debug('all_settings_here=%s' % all_settings)
system_id = int(getval(run_settings, '%s/system/id' % self.SCHEMA_PREFIX)) #TODO Mytardis
experiment_id = mytardis.create_dataset(
settings=all_settings,
source_url=source_dir_url,
exp_id=experiment_id,
exp_name=mytardis.get_exp_name_for_intermediate_output,
dataset_name=mytardis.get_dataset_name_for_output,
dataset_paramset=dataset_paramset,
datafile_paramset=datafile_paramset,
dfile_extract_func=dfile_extract_func
)
else:
logger.warn("no mytardis host specified")
return 0
return experiment_id
def process_outputs(self, run_settings, base_dir, input_url, all_settings):
id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
iter_output_dir = os.path.join(os.path.join(base_dir, "input_%s" % (id + 1)))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
(scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(input_url)
iter_out_fsys = storage.get_filesystem(input_url)
input_dirs, _ = iter_out_fsys.listdir(iter_output_path)
# TODO: store all audit info in single file in input_X directory in transform,
# so we do not have to load individual files within node directories here.
min_crit = sys.float_info.max - 1.0
min_crit_index = sys.maxint
# # TODO: store all audit info in single file in input_X directory in transform,
# # so we do not have to load individual files within node directories here.
# min_crit = sys.float_info.max - 1.0
# min_crit_index = sys.maxint
logger.debug("input_dirs=%s" % input_dirs)
for input_dir in input_dirs:
node_path = os.path.join(iter_output_dir, input_dir)
logger.debug('node_path= %s' % node_path)
# Retrieve audit file
# audit_url = get_url_with_credentials(output_storage_settings,
# output_prefix + os.path.join(self.iter_inputdir, input_dir, 'audit.txt'), is_relative_path=False)
audit_url = get_url_with_credentials(all_settings, os.path.join(node_path, "audit.txt"), is_relative_path=False)
audit_content = storage.get_file(audit_url)
logger.debug('audit_url=%s' % audit_url)
# extract the best criterion error
# FIXME: audit.txt is potentially debug file so format may not be fixed.
p = re.compile("Run (\d+) preserved \(error[ \t]*([0-9\.]+)\)", re.MULTILINE)
m = p.search(audit_content)
criterion = None
if m:
criterion = float(m.group(2))
best_numb = int(m.group(1))
# NB: assumes that subdirss in new input_x will have same names as output dir that created it.
best_node = input_dir
else:
message = "Cannot extract criterion from audit file for iteration %s" % (self.id + 1)
logger.warn(message)
raise IOError(message)
if criterion < min_crit:
min_crit = criterion
min_crit_index = best_numb
min_crit_node = best_node
logger.debug("min_crit = %s at %s" % (min_crit, min_crit_index))
if min_crit_index >= sys.maxint:
raise BadInputException("Unable to find minimum criterion of input files")
# get previous best criterion
try:
self.prev_criterion = float(getval(run_settings, '%s/converge/criterion' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.prev_criterion = sys.float_info.max - 1.0
logger.warn("no previous criterion found")
# check whether we are under the error threshold
logger.debug("best_num=%s" % best_numb)
logger.debug("prev_criterion = %f" % self.prev_criterion)
logger.debug("min_crit = %f" % min_crit)
logger.debug('Current min criterion: %f, Prev '
'criterion: %f' % (min_crit, self.prev_criterion))
difference = self.prev_criterion - min_crit
logger.debug("Difference %f" % difference)
try:
max_iteration = int(getval(run_settings, '%s/input/hrmc/max_iteration' % RMIT_SCHEMA))
except (ValueError, SettingNotFoundException):
raise BadInputException("unknown max_iteration")
logger.debug("max_iteration=%s" % max_iteration)
try:
self.error_threshold = float(getval(run_settings, '%s/input/hrmc/error_threshold' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
raise BadInputException("uknown error threshold")
logger.debug("error_threshold=%s" % self.error_threshold)
if self.id >= (max_iteration - 1):
logger.debug("Max Iteration Reached %d " % self.id)
return (True, min_crit)
elif min_crit <= self.prev_criterion and difference <= self.error_threshold:
logger.debug("Convergence reached %f" % difference)
return (True, min_crit)
else:
if difference < 0:
logger.debug("iteration diverged")
logger.debug("iteration continues: %d iteration so far" % self.id)
return (False, min_crit)
def curate_dataset(self, run_settings, experiment_id,
base_url, output_url, all_settings):
'''
Curates dataset
'''
# Retrieves process directories below the current output location
iteration = int(getval(run_settings, '%s/system/id' % SCHEMA_PREFIX))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
current_output_url = "%s%s" % (output_prefix, os.path.join(os.path.join(
base_url, "output_%s" % iteration)))
(scheme, host, current_output_path, location, query_settings) = storage.parse_bdpurl(output_url)
output_fsys = storage.get_filesystem(output_url)
process_output_dirs, _ = output_fsys.listdir(current_output_path)
# Curates a dataset with metadata per process
for i, process_output_dir in enumerate(process_output_dirs):
# Expand the process output directory and add credentials for access
process_output_url = '/'.join([current_output_url, process_output_dir])
process_output_url_with_cred = get_url_with_credentials(
all_settings,
process_output_url,
is_relative_path=False)
# Expand the process output file and add credentials for access
output_file_url_with_cred = storage.get_url_with_credentials(
all_settings, '/'.join([process_output_url, OUTPUT_FILE]),
is_relative_path=False)
try:
output_content = storage.get_file(output_file_url_with_cred)
val1, val2 = output_content.split()
except (IndexError, IOError) as e:
logger.warn(e)
continue
try:
x = float(val1)
y = float(val2)
except (ValueError, IndexError) as e:
logger.warn(e)
continue
# Returns the process id as MyTardis dataset name
all_settings['graph_point_id'] = str(i)
def _get_dataset_name(settings, url, path):
return all_settings['graph_point_id']
# Creates new dataset and adds to experiment
# If experiment_id==0, creates new experiment
experiment_id = mytardis.create_dataset(
settings=all_settings, # MyTardis credentials
source_url=process_output_url_with_cred,
exp_id=experiment_id,
dataset_name=_get_dataset_name, # the function that defines dataset name
dataset_paramset=[
# a new blank parameter set conforming to schema 'remotemake/output'
mytardis.create_paramset("remotemake/output", []),
mytardis.create_graph_paramset("dsetgraph", # name of schema
name="randdset", # a unique dataset name
graph_info={},
value_dict={"randdset/x": x, "randdset/y": y}, # values to be used in experiment graphs
value_keys=[]
),
]
)
return experiment_id
def curate_dataset(self, run_settings, experiment_id, base_url, output_url,
all_settings):
'''
Curates dataset
'''
# Retrieves process directories below the current output location
iteration = int(getval(run_settings, '%s/system/id' % SCHEMA_PREFIX))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
current_output_url = "%s%s" % (
output_prefix,
os.path.join(os.path.join(base_url, "output_%s" % iteration)))
(scheme, host, current_output_path, location,
query_settings) = storage.parse_bdpurl(output_url)
output_fsys = storage.get_filesystem(output_url)
process_output_dirs, _ = output_fsys.listdir(current_output_path)
# Curates a dataset with metadata per process
for i, process_output_dir in enumerate(process_output_dirs):
# Expand the process output directory and add credentials for access
process_output_url = '/'.join(
[current_output_url, process_output_dir])
process_output_url_with_cred = get_url_with_credentials(
all_settings, process_output_url, is_relative_path=False)
# Expand the process output file and add credentials for access
output_file_url_with_cred = storage.get_url_with_credentials(
all_settings,
'/'.join([process_output_url, OUTPUT_FILE]),
is_relative_path=False)
try:
output_content = storage.get_file(output_file_url_with_cred)
val1, val2 = output_content.split()
except (IndexError, IOError) as e:
logger.warn(e)
continue
try:
x = float(val1)
y = float(val2)
except (ValueError, IndexError) as e:
logger.warn(e)
continue
# Returns the process id as MyTardis dataset name
all_settings['graph_point_id'] = str(i)
def _get_dataset_name(settings, url, path):
return all_settings['graph_point_id']
# Creates new dataset and adds to experiment
# If experiment_id==0, creates new experiment
experiment_id = mytardis.create_dataset(
settings=all_settings, # MyTardis credentials
source_url=process_output_url_with_cred,
exp_id=experiment_id,
dataset_name=
_get_dataset_name, # the function that defines dataset name
dataset_paramset=[
# a new blank parameter set conforming to schema 'remotemake/output'
mytardis.create_paramset("remotemake/output", []),
mytardis.create_graph_paramset(
"dsetgraph", # name of schema
name="randdset", # a unique dataset name
graph_info={},
value_dict={
"randdset/x": x,
"randdset/y": y
}, # values to be used in experiment graphs
value_keys=[]),
])
return experiment_id
def process_outputs(self, run_settings, base_dir, output_url, all_settings, offset):
# output_dir = 118.138.241.232/outptuersdfsd/sweep277/hrmc278/output_1
# output_prefix = ssh://unix@
# node_output_dir = 2
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
iter_output_dir = os.path.join(os.path.join(base_dir, "output_%s" % id))
logger.debug('iter_output_dir=%s' % iter_output_dir)
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
logger.debug('output_prefix=%s' % output_prefix)
#iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
logger.debug('output_url=%s' % output_url)
(scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(output_url)
logger.debug("iter_output_path=%s" % iter_output_path)
iter_out_fsys = storage.get_filesystem(output_url)
logger.debug("iter_out_fsys=%s" % iter_out_fsys)
node_output_dirnames, _ = iter_out_fsys.listdir(iter_output_path)
logger.debug('node_output_dirnames=%s' % node_output_dirnames)
self.audit = ""
Node_info = namedtuple('Node_info',
['dirname', 'number', 'criterion'])
BASE_FNAME = "HRMC.inp"
# generate criterias
self.outputs = []
for node_output_dirname in node_output_dirnames:
node_path = output_prefix + os.path.join(iter_output_dir, node_output_dirname)
criterion = self.compute_psd_criterion(all_settings, node_path)
#criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dirname, fs,)
logger.debug("criterion=%s" % criterion)
try:
values_url = get_url_with_credentials(
all_settings, os.path.join(node_path,
'%s_values' % BASE_FNAME), is_relative_path=False)
values_content = storage.get_file(values_url)
logger.debug("values_file=%s" % values_url)
except IOError:
logger.warn("no values file found")
values_map = {}
else:
values_map = dict(json.loads(values_content))
self.outputs.append(Node_info(dirname=node_output_dirname,
number=values_map['run_counter'], criterion=criterion))
if not self.outputs:
logger.error("no ouput found for this iteration")
return
self.outputs.sort(key=lambda x: int(x.criterion))
logger.debug("self.outputs=%s" % self.outputs)
try:
# FIXME: need to validate this output to make sure list of int
threshold = ast.literal_eval(getval(run_settings, '%s/input/hrmc/threshold' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
logger.warn("no threshold found when expected")
return False
logger.debug("threshold = %s" % threshold)
total_picks = 1
if len(threshold) > 1:
for i in threshold:
total_picks *= threshold[i]
else:
total_picks = threshold[0]
def copy_files_with_pattern(iter_out_fsys, source_path,
dest_path, pattern, all_settings):
"""
"""
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
logger.debug('source_path=%s, dest_path=%s' % (source_path, dest_path))
# (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(source_path)
_, node_output_fnames = iter_out_fsys.listdir(source_path)
ip_address = all_settings['ip_address']
for f in node_output_fnames:
if fnmatch.fnmatch(f, pattern):
source_url = get_url_with_credentials(all_settings, output_prefix + os.path.join(ip_address, source_path, f), is_relative_path=False)
dest_url = get_url_with_credentials(all_settings, output_prefix + os.path.join(ip_address, dest_path, f), is_relative_path=False)
logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url))
content = storage.get_file(source_url)
storage.put_file(dest_url, content)
# Make new input dirs
new_input_dir = os.path.join(os.path.join(base_dir, "input_%d" % (id + 1)))
for index in range(0, total_picks):
Node_info = self.outputs[index]
logger.debug("node_info.dirname=%s" % Node_info.dirname)
logger.debug("Node_info=%s" % str(Node_info))
new_input_path = os.path.join(new_input_dir,
Node_info.dirname)
logger.debug("New input node dir %s" % new_input_path)
old_output_path = os.path.join(iter_output_dir, Node_info.dirname)
# Move all existing domain input files unchanged to next input directory
for f in DOMAIN_INPUT_FILES:
source_url = get_url_with_credentials(
all_settings, output_prefix + os.path.join(old_output_path, f), is_relative_path=False)
dest_url = get_url_with_credentials(
all_settings, output_prefix + os.path.join(new_input_path, f),
is_relative_path=False)
logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url))
content = storage.get_file(source_url)
logger.debug('content collected')
storage.put_file(dest_url, content)
logger.debug('put successfully')
logger.debug('put file successfully')
pattern = "*_values"
output_offset = os.path.join(os.path.join(offset, "output_%s" % id, Node_info.dirname))
input_offset = os.path.join(os.path.join(offset, "input_%s" % (id + 1), Node_info.dirname))
copy_files_with_pattern(iter_out_fsys,
output_offset,
input_offset, pattern,
all_settings)
pattern = "*_template"
copy_files_with_pattern(iter_out_fsys,
output_offset,
input_offset, pattern,
all_settings)
# NB: Converge stage triggers based on criterion value from audit.
logger.debug('starting audit')
info = "Run %s preserved (error %s)\n" % (Node_info.number, Node_info.criterion)
audit_url = get_url_with_credentials(
all_settings, output_prefix +
os.path.join(new_input_path, 'audit.txt'), is_relative_path=False)
storage.put_file(audit_url, info)
logger.debug("audit=%s" % info)
logger.debug('1:audit_url=%s' % audit_url)
self.audit += info
# move xyz_final.xyz to initial.xyz
source_url = get_url_with_credentials(
all_settings, output_prefix + os.path.join(old_output_path, "xyz_final.xyz"), is_relative_path=False)
logger.debug('source_url=%s' % source_url)
dest_url = get_url_with_credentials(
all_settings, output_prefix + os.path.join(new_input_path, 'input_initial.xyz'), is_relative_path=False)
logger.debug('dest_url=%s' % dest_url)
content = storage.get_file(source_url)
logger.debug('content=%s' % content)
storage.put_file(dest_url, content)
self.audit += "spawning diamond runs\n"
logger.debug("input_dir=%s" % (output_prefix + os.path.join(new_input_dir, 'audit.txt')))
audit_url = get_url_with_credentials(
all_settings, output_prefix + os.path.join(new_input_dir, 'audit.txt'), is_relative_path=False)
logger.debug('audit_url=%s' % audit_url)
storage.put_file(audit_url, self.audit)
def curate_dataset(self, run_settings, experiment_id, base_dir, output_url,
all_settings):
iteration = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
iter_output_dir = os.path.join(os.path.join(base_dir, "output_%s" % iteration))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
(scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
fsys = storage.get_filesystem(output_url)
node_output_dirnames, _ = fsys.listdir(mypath)
logger.debug("node_output_dirnames=%s" % node_output_dirnames)
if all_settings['mytardis_host']:
for i, node_output_dirname in enumerate(node_output_dirnames):
node_path = os.path.join(iter_output_dir, node_output_dirname)
# find criterion
crit = None # is there an infinity criterion
for ni in self.outputs:
if ni.dirname == node_output_dirname:
crit = ni.criterion
break
else:
logger.debug("criterion not found")
continue
logger.debug("crit=%s" % crit)
# graph_params = []
def extract_psd_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
res = {"hrmcdfile/r1": xs, "hrmcdfile/g1": ys}
return res
def extract_psdexp_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
res = {"hrmcdfile/r2": xs, "hrmcdfile/g2": ys}
return res
def extract_grfinal_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
#FIXME: len(xs) == len(ys) for this to work.
#TODO: hack to handle when xs and ys are too
# large to fit in Parameter with db_index.
# solved by function call at destination
cut_xs = [xs[i] for i, x in enumerate(xs)
if (i % (len(xs) / 20) == 0)]
cut_ys = [ys[i] for i, x in enumerate(ys)
if (i % (len(ys) / 20) == 0)]
res = {"hrmcdfile/r3": cut_xs, "hrmcdfile/g3": cut_ys}
return res
def extract_inputgr_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
#FIXME: len(xs) == len(ys) for this to work.
#TODO: hack to handle when xs and ys are too
# large to fit in Parameter with db_index.
# solved by function call at destination
cut_xs = [xs[i] for i, x in enumerate(xs)
if (i % (len(xs) / 20) == 0)]
cut_ys = [ys[i] for i, x in enumerate(ys)
if (i % (len(ys) / 20) == 0)]
res = {"hrmcdfile/r4": cut_xs, "hrmcdfile/g4": cut_ys}
return res
#TODO: hrmcexp graph should be tagged to input directories (not output directories)
#because we want the result after pruning.
#todo: replace self.boto_setttings with mytardis_settings
EXP_DATASET_NAME_SPLIT = 2
def get_exp_name_for_output(settings, url, path):
# return str(os.sep.join(path.split(os.sep)[:-EXP_DATASET_NAME_SPLIT]))
return str(os.sep.join(path.split(os.sep)[-4:-2]))
def get_dataset_name_for_output(settings, url, path):
logger.debug("path=%s" % path)
host = settings['host']
prefix = 'ssh://%s@%s' % (settings['type'], host)
source_url = get_url_with_credentials(
settings, os.path.join(prefix, path, "HRMC.inp_values"),
is_relative_path=False)
logger.debug("source_url=%s" % source_url)
try:
content = storage.get_file(source_url)
except IOError, e:
logger.warn("cannot read file %s" % e)
return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
logger.debug("content=%s" % content)
try:
values_map = dict(json.loads(str(content)))
except Exception, e:
logger.warn("cannot load %s: %s" % (content, e))
return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
try:
iteration = str(path.split(os.sep)[-2:-1][0])
except Exception, e:
logger.error(e)
iteration = ""
if "_" in iteration:
iteration = iteration.split("_")[1]
else:
iteration = "final"
dataset_name = "%s_%s_%s" % (iteration,
values_map['generator_counter'],
values_map['run_counter'])
logger.debug("dataset_name=%s" % dataset_name)
return dataset_name
def process(self, run_settings):
"""
Check all registered nodes to find whether
they are running, stopped or in error_nodes
"""
local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)
# local_settings = run_settings[models.UserProfile.PROFILE_SCHEMA_NS]
retrieve_local_settings(run_settings, local_settings)
logger.debug("local_settings=%s" % local_settings)
self.contextid = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX)
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX)
output_storage_settings = get_platform_settings(output_storage_url, local_settings['bdp_username'])
# FIXME: Need to be consistent with how we handle settings here. Prob combine all into
# single local_settings for simplicity.
output_storage_settings['bdp_username'] = local_settings['bdp_username']
offset = getval(run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX)
self.job_dir = get_job_dir(output_storage_settings, offset)
try:
self.finished_nodes = getval(run_settings, '%s/stages/run/finished_nodes' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
self.finished_nodes = '[]'
try:
self.id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX))
self.output_dir = "output_%s" % self.id
except (SettingNotFoundException, ValueError):
self.id = 0
self.output_dir = "output"
logger.debug("output_dir=%s" % self.output_dir)
logger.debug("run_settings=%s" % run_settings)
logger.debug("Wait stage process began")
#processes = self.executed_procs
processes = [x for x in self.current_processes if x['status'] == 'running']
self.error_nodes = []
# TODO: parse finished_nodes input
logger.debug('self.finished_nodes=%s' % self.finished_nodes)
self.finished_nodes = ast.literal_eval(self.finished_nodes)
computation_platform_url = getval(run_settings, '%s/platform/computation/platform_url' % django_settings.SCHEMA_PREFIX)
comp_pltf_settings = get_platform_settings(computation_platform_url, local_settings['bdp_username'])
local_settings.update(comp_pltf_settings)
comp_pltf_settings['bdp_username'] = local_settings['bdp_username']
wait_strategy = strategies.SynchronousWaitStrategy()
try:
payload_source = getval(run_settings, '%s/stages/setup/payload_source' % django_settings.SCHEMA_PREFIX)
if payload_source:
wait_strategy = strategies.AsynchronousWaitStrategy()
except SettingNotFoundException:
pass
for process in processes:
#instance_id = node.id
ip_address = process['ip_address']
process_id = process['id']
retry_left = process['retry_left']
#ip = botocloudconnector.get_instance_ip(instance_id, self.boto_settings)
#ssh = open_connection(ip_address=ip, settings=self.boto_settings)
#if not botocloudconnector.is_vm_running(node):
# An unlikely situation where the node crashed after is was
# detected as registered.
#FIXME: should error nodes be counted as finished?
# logging.error('Instance %s not running' % instance_id)
# self.error_nodes.append(node)
# continue
relative_path_suffix = self.get_relative_output_path(local_settings)
fin = wait_strategy.is_job_finished( self,
ip_address, process_id, retry_left,
local_settings, relative_path_suffix)
logger.debug("fin=%s" % fin)
if fin:
logger.debug("done. output is available")
logger.debug("node=%s" % str(process))
logger.debug("finished_nodes=%s" % self.finished_nodes)
#FIXME: for multiple nodes, if one finishes before the other then
#its output will be retrieved, but it may again when the other node fails, because
#we cannot tell whether we have prevous retrieved this output before and finished_nodes
# is not maintained between triggerings...
if not (int(process_id) in [int(x['id'])
for x in self.finished_nodes
if int(process_id) == int(x['id'])]):
self.get_output(ip_address, process_id, self.output_dir,
local_settings, comp_pltf_settings,
output_storage_settings, run_settings)
audit_url = get_url_with_credentials(
comp_pltf_settings, os.path.join(
self.output_dir, process_id, "audit.txt"),
is_relative_path=True)
fsys = storage.get_filesystem(audit_url)
logger.debug("Audit file url %s" % audit_url)
if fsys.exists(audit_url):
fsys.delete(audit_url)
self.finished_nodes.append(process)
logger.debug('finished_processes=%s' % self.finished_nodes)
for iterator, p in enumerate(self.all_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.all_processes[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.executed_procs):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.executed_procs[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.current_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.current_processes[iterator]['status'] = 'completed'
else:
logger.warn("We have already "
+ "processed output of %s on node %s" % (process_id, ip_address))
else:
print "job %s at %s not completed" % (process_id, ip_address)
failed_processes = [x for x in self.current_processes if x['status'] == 'failed']
logger.debug('failed_processes=%s' % failed_processes)
logger.debug('failed_processes=%d' % len(failed_processes))
messages.info(run_settings, "%d: Waiting %d processes (%d completed, %d failed) " % (
self.id + 1, len(self.current_processes), len(self.finished_nodes),
len(failed_processes)))
def process(self, run_settings):
"""
Check all registered nodes to find whether
they are running, stopped or in error_nodes
"""
local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)
# local_settings = run_settings[models.UserProfile.PROFILE_SCHEMA_NS]
retrieve_local_settings(run_settings, local_settings)
logger.debug("local_settings=%s" % local_settings)
self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA)
output_storage_settings = get_platform_settings(output_storage_url, local_settings['bdp_username'])
# FIXME: Need to be consistent with how we handle settings here. Prob combine all into
# single local_settings for simplicity.
output_storage_settings['bdp_username'] = local_settings['bdp_username']
offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA)
self.job_dir = get_job_dir(output_storage_settings, offset)
try:
self.finished_nodes = getval(run_settings, '%s/stages/run/finished_nodes' % RMIT_SCHEMA)
# self.finished_nodes = smartconnectorscheduler.get_existing_key(run_settings,
# 'http://rmit.edu.au/schemas/stages/run/finished_nodes')
except SettingNotFoundException:
self.finished_nodes = '[]'
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
# self.id = int(smartconnectorscheduler.get_existing_key(run_settings,
# 'http://rmit.edu.au/schemas/system/id'))
self.output_dir = "output_%s" % self.id
except (SettingNotFoundException, ValueError):
self.id = 0
self.output_dir = "output"
logger.debug("output_dir=%s" % self.output_dir)
logger.debug("run_settings=%s" % run_settings)
logger.debug("Wait stage process began")
#processes = self.executed_procs
processes = [x for x in self.current_processes if x['status'] == 'running']
self.error_nodes = []
# TODO: parse finished_nodes input
logger.debug('self.finished_nodes=%s' % self.finished_nodes)
self.finished_nodes = ast.literal_eval(self.finished_nodes)
computation_platform_url = getval(run_settings, '%s/platform/computation/platform_url' % RMIT_SCHEMA)
comp_pltf_settings = get_platform_settings(computation_platform_url, local_settings['bdp_username'])
local_settings.update(comp_pltf_settings)
comp_pltf_settings['bdp_username'] = local_settings['bdp_username']
wait_strategy = strategies.SynchronousWaitStrategy()
try:
synchronous_wait = getval(run_settings, '%s/stages/wait/synchronous' % RMIT_SCHEMA)
if not synchronous_wait:
wait_strategy = strategies.AsynchronousWaitStrategy()
except SettingNotFoundException:
pass
for process in processes:
#instance_id = node.id
ip_address = process['ip_address']
process_id = process['id']
retry_left = process['retry_left']
#ip = botocloudconnector.get_instance_ip(instance_id, self.boto_settings)
#ssh = open_connection(ip_address=ip, settings=self.boto_settings)
#if not botocloudconnector.is_vm_running(node):
# An unlikely situation where the node crashed after is was
# detected as registered.
#FIXME: should error nodes be counted as finished?
# logging.error('Instance %s not running' % instance_id)
# self.error_nodes.append(node)
# continue
relative_path_suffix = self.get_relative_output_path(local_settings)
fin = wait_strategy.is_job_finished( self,
ip_address, process_id, retry_left,
local_settings, relative_path_suffix)
logger.debug("fin=%s" % fin)
if fin:
logger.debug("done. output is available")
logger.debug("node=%s" % str(process))
logger.debug("finished_nodes=%s" % self.finished_nodes)
#FIXME: for multiple nodes, if one finishes before the other then
#its output will be retrieved, but it may again when the other node fails, because
#we cannot tell whether we have prevous retrieved this output before and finished_nodes
# is not maintained between triggerings...
if not (int(process_id) in [int(x['id'])
for x in self.finished_nodes
if int(process_id) == int(x['id'])]):
self.get_output(ip_address, process_id, self.output_dir,
local_settings, comp_pltf_settings,
output_storage_settings, run_settings)
audit_url = get_url_with_credentials(
comp_pltf_settings, os.path.join(
self.output_dir, process_id, "audit.txt"),
is_relative_path=True)
fsys = storage.get_filesystem(audit_url)
logger.debug("Audit file url %s" % audit_url)
if fsys.exists(audit_url):
fsys.delete(audit_url)
self.finished_nodes.append(process)
logger.debug('finished_processes=%s' % self.finished_nodes)
for iterator, p in enumerate(self.all_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.all_processes[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.executed_procs):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.executed_procs[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.current_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.current_processes[iterator]['status'] = 'completed'
else:
logger.warn("We have already "
+ "processed output of %s on node %s" % (process_id, ip_address))
else:
print "job %s at %s not completed" % (process_id, ip_address)
failed_processes = [x for x in self.current_processes if x['status'] == 'failed']
logger.debug('failed_processes=%s' % failed_processes)
logger.debug('failed_processes=%d' % len(failed_processes))
messages.info(run_settings, "%d: waiting %d processes (%d completed, %d failed) " % (
self.id + 1, len(self.current_processes), len(self.finished_nodes),
len(failed_processes)))
def create_dataset_for_final_output(self, run_settings, experiment_id,
base_dir, output_url, all_settings):
logger.debug("curate_dataset")
iter_output_dir = os.path.join(os.path.join(base_dir, "output"))
logger.debug("iter_output_dir=%s" % iter_output_dir)
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
logger.debug("iter_output_dir=%s" % iter_output_dir)
logger.debug("output_url=%s" % output_url)
(scheme, host, mypath, location,
query_settings) = storage.parse_bdpurl(output_url)
fsys = storage.get_filesystem(output_url)
node_output_dirnames, _ = fsys.listdir(mypath)
logger.debug("node_output_dirnames=%s" % node_output_dirnames)
curate_data = (getval(
run_settings,
'%s/input/mytardis/curate_data' % self.SCHEMA_PREFIX))
if curate_data:
if all_settings['mytardis_host']:
output_dirs = []
for m, dir_name in enumerate(node_output_dirnames):
output_dirs.append(os.path.join(iter_output_dir, dir_name))
for m, output_dir in enumerate(output_dirs):
#node_path = os.path.join(iter_output_dir, node_dir)
logger.debug("output_dir=%s" % output_dir)
dataset_paramset = []
datafile_paramset = []
dfile_extract_func = {}
self.load_metadata_builder(run_settings)
if self.METADATA_BUILDER:
(experiment_paramset, dataset_paramset, datafile_paramset, dfile_extract_func) = \
self.METADATA_BUILDER.build_metadata_for_final_output(m, output_dir, \
run_settings=run_settings, storage_settings=all_settings,\
output_dirs=output_dirs)
source_url = get_url_with_credentials(
all_settings, output_dir, is_relative_path=False)
logger.debug("source_url=%s" % source_url)
experiment_id = mytardis.create_dataset(
settings=all_settings,
source_url=source_url,
exp_name=mytardis.get_exp_name_for_output,
dataset_name=mytardis.get_dataset_name_for_output,
exp_id=experiment_id,
experiment_paramset=experiment_paramset,
dataset_paramset=dataset_paramset,
datafile_paramset=datafile_paramset,
dfile_extract_func=dfile_extract_func)
graph_paramset = []
else:
logger.warn("no mytardis host specified")
else:
logger.warn('Data curation is off')
return experiment_id
def create_dataset_for_intermediate_output(self,
run_settings,
experiment_id,
base_dir,
output_url,
all_settings,
outputs=[]):
logger.debug('self_outpus_curate=%s' % outputs)
iteration = int(
getval(run_settings, '%s/system/id' % self.SCHEMA_PREFIX))
iter_output_dir = os.path.join(
os.path.join(base_dir, "output_%s" % iteration))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
(scheme, host, mypath, location,
query_settings) = storage.parse_bdpurl(output_url)
fsys = storage.get_filesystem(output_url)
node_output_dirnames, _ = fsys.listdir(mypath)
logger.debug("node_output_dirnames=%s" % node_output_dirnames)
if all_settings['mytardis_host']:
output_dirs = []
for m, dir_name in enumerate(node_output_dirnames):
output_dirs.append(os.path.join(iter_output_dir, dir_name))
for i, output_dir in enumerate(output_dirs):
dataset_paramset = []
datafile_paramset = []
dfile_extract_func = {}
self.load_metadata_builder(run_settings)
if self.METADATA_BUILDER:
(continue_loop, dataset_paramset, datafile_paramset, dfile_extract_func) = \
self.METADATA_BUILDER.build_metadata_for_intermediate_output(\
output_dir, outputs, run_settings=run_settings, storage_settings=all_settings,\
output_dirs=output_dirs)
if continue_loop:
continue
source_dir_url = get_url_with_credentials(
all_settings, output_dir, is_relative_path=False)
logger.debug("source_dir_url=%s" % source_dir_url)
logger.debug('all_settings_here=%s' % all_settings)
system_id = int(
getval(run_settings, '%s/system/id' %
self.SCHEMA_PREFIX)) #TODO Mytardis
experiment_id = mytardis.create_dataset(
settings=all_settings,
source_url=source_dir_url,
exp_id=experiment_id,
exp_name=mytardis.get_exp_name_for_intermediate_output,
dataset_name=mytardis.get_dataset_name_for_output,
dataset_paramset=dataset_paramset,
datafile_paramset=datafile_paramset,
dfile_extract_func=dfile_extract_func)
else:
logger.warn("no mytardis host specified")
return 0
return experiment_id
def curate_dataset(self, run_settings, experiment_id, base_dir, output_url, all_settings):
logger.debug("curate_dataset")
iter_output_dir = os.path.join(os.path.join(base_dir, "output"))
logger.debug("iter_output_dir=%s" % iter_output_dir)
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
logger.debug("iter_output_dir=%s" % iter_output_dir)
logger.debug("output_url=%s" % output_url)
(scheme, host, mypath, location, query_settings) = storage.parse_bdpurl(output_url)
fsys = storage.get_filesystem(output_url)
node_output_dirnames, _ = fsys.listdir(mypath)
logger.debug("node_output_dirnames=%s" % node_output_dirnames)
curate_data = (getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA))
if curate_data:
if all_settings['mytardis_host']:
# if mytardis_settings['mytardis_host']:
# EXP_DATASET_NAME_SPLIT = 2
# def get_exp_name_for_output(settings, url, path):
# return str(os.sep.join(path.split(os.sep)[:-EXP_DATASET_NAME_SPLIT]))
# def get_dataset_name_for_output(settings, url, path):
# logger.debug("path=%s" % path)
# host = settings['host']
# prefix = 'ssh://%s@%s' % (settings['type'], host)
# source_url = smartconnectorscheduler.get_url_with_credentials(
# settings, os.path.join(prefix, path, "HRMC.inp_values"),
# is_relative_path=False)
# logger.debug("source_url=%s" % source_url)
# try:
# content = storage.get_file(source_url)
# except IOError, e:
# logger.warn("cannot read file %s" % e)
# return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
# logger.debug("content=%s" % content)
# try:
# values_map = dict(json.loads(str(content)))
# except Exception, e:
# logger.error("cannot load values_map %s: from %s. Error=%s" % (content, source_url, e))
# return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
# try:
# iteration = str(path.split(os.sep)[-2:-1][0])
# except Exception, e:
# logger.error(e)
# iteration = ""
# if "_" in iteration:
# iteration = iteration.split("_")[1]
# else:
# iteration = "final"
# dataset_name = "%s_%s_%s" % (iteration,
# values_map['generator_counter'],
# values_map['run_counter'])
# logger.debug("dataset_name=%s" % dataset_name)
# return dataset_name
# re_dbl_fort = re.compile(r'(\d*\.\d+)[dD]([-+]?\d+)')
# logger.debug("new_output_dir=%s" % new_output_dir)
# exp_value_keys = []
# legends = []
# for m, node_dir in enumerate(node_dirs):
# exp_value_keys.append(["hrmcdset%s/step" % m, "hrmcdset%s/err" % m])
# source_url = smartconnectorscheduler.get_url_with_credentials(output_storage_settings,
# output_prefix + os.path.join(new_output_dir, node_dir), is_relative_path=False)
# (source_scheme, source_location, source_path, source_location,
# query_settings) = storage.parse_bdpurl(source_url)
# logger.debug("source_url=%s" % source_url)
# legends.append(
# get_dataset_name_for_output(
# output_storage_settings, "", source_path))
# logger.debug("exp_value_keys=%s" % exp_value_keys)
# logger.debug("legends=%s" % legends)
# graph_paramset = [mytardis.create_graph_paramset("expgraph",
# name="hrmcexp2",
# graph_info={"axes": ["step", "ERRGr*wf"], "precision": [0, 2], "legends": legends},
# value_dict={},
# value_keys=exp_value_keys)]
# for m, node_dir in enumerate(node_dirs):
# dataerrors_url = smartconnectorscheduler.get_url_with_credentials(output_storage_settings,
# output_prefix + os.path.join(new_output_dir, node_dir, DATA_ERRORS_FILE), is_relative_path=False)
# dataerrors_content = storage.get_file(dataerrors_url)
# xs = []
# ys = []
# for i, line in enumerate(dataerrors_content.splitlines()):
# if i == 0:
# continue
# columns = line.split()
# try:
# hrmc_step = int(columns[STEP_COLUMN_NUM])
# except ValueError:
# logger.warn("could not parse hrmc_step value on line %s" % i)
# continue
# # handle format double precision float format
# val = columns[ERRGR_COLUMN_NUM]
# val = re_dbl_fort.sub(r'\1E\2', val)
# logger.debug("val=%s" % val)
EXP_DATASET_NAME_SPLIT = 2
def get_exp_name_for_output(settings, url, path):
return str(os.sep.join(path.split(os.sep)[:-EXP_DATASET_NAME_SPLIT]))
def get_dataset_name_for_output(settings, url, path):
logger.debug("path=%s" % path)
host = settings['host']
prefix = 'ssh://%s@%s' % (settings['type'], host)
source_url = get_url_with_credentials(
settings, os.path.join(prefix, path, "HRMC.inp_values"),
is_relative_path=False)
logger.debug("source_url=%s" % source_url)
try:
content = storage.get_file(source_url)
except IOError, e:
logger.warn("cannot read file %s" % e)
return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
logger.debug("content=%s" % content)
try:
values_map = dict(json.loads(str(content)))
except Exception, e:
logger.error("cannot load values_map %s: from %s. Error=%s" % (content, source_url, e))
return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
try:
iteration = str(path.split(os.sep)[-2:-1][0])
except Exception, e:
logger.error(e)
iteration = ""
if "_" in iteration:
iteration = iteration.split("_")[1]
else:
iteration = "final"
dataset_name = "%s_%s_%s" % (iteration,
values_map['generator_counter'],
values_map['run_counter'])
logger.debug("dataset_name=%s" % dataset_name)
return dataset_name
def process_outputs(self, run_settings, base_dir, input_url, all_settings):
id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
iter_output_dir = os.path.join(os.path.join(base_dir, "input_%s" % (id + 1)))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
(scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(input_url)
iter_out_fsys = storage.get_filesystem(input_url)
input_dirs, _ = iter_out_fsys.listdir(iter_output_path)
# TODO: store all audit info in single file in input_X directory in transform,
# so we do not have to load individual files within node directories here.
min_crit = sys.float_info.max - 1.0
min_crit_index = sys.maxint
# # TODO: store all audit info in single file in input_X directory in transform,
# # so we do not have to load individual files within node directories here.
# min_crit = sys.float_info.max - 1.0
# min_crit_index = sys.maxint
logger.debug("input_dirs=%s" % input_dirs)
for input_dir in input_dirs:
node_path = os.path.join(iter_output_dir, input_dir)
logger.debug('node_path= %s' % node_path)
# Retrieve audit file
# audit_url = get_url_with_credentials(output_storage_settings,
# output_prefix + os.path.join(self.iter_inputdir, input_dir, 'audit.txt'), is_relative_path=False)
audit_url = get_url_with_credentials(all_settings, os.path.join(node_path, "audit.txt"), is_relative_path=False)
audit_content = storage.get_file(audit_url)
logger.debug('audit_url=%s' % audit_url)
# extract the best criterion error
# FIXME: audit.txt is potentially debug file so format may not be fixed.
p = re.compile("Run (\d+) preserved \(error[ \t]*([0-9\.]+)\)", re.MULTILINE)
m = p.search(audit_content)
criterion = None
if m:
criterion = float(m.group(2))
best_numb = int(m.group(1))
# NB: assumes that subdirss in new input_x will have same names as output dir that created it.
best_node = input_dir
else:
message = "Cannot extract criterion from audit file for iteration %s" % (self.id + 1)
logger.warn(message)
raise IOError(message)
if criterion < min_crit:
min_crit = criterion
min_crit_index = best_numb
min_crit_node = best_node
logger.debug("min_crit = %s at %s" % (min_crit, min_crit_index))
if min_crit_index >= sys.maxint:
raise BadInputException("Unable to find minimum criterion of input files")
# get previous best criterion
try:
self.prev_criterion = float(getval(run_settings, '%s/converge/criterion' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.prev_criterion = sys.float_info.max - 1.0
logger.warn("no previous criterion found")
# check whether we are under the error threshold
logger.debug("best_num=%s" % best_numb)
logger.debug("prev_criterion = %f" % self.prev_criterion)
logger.debug("min_crit = %f" % min_crit)
logger.debug('Current min criterion: %f, Prev '
'criterion: %f' % (min_crit, self.prev_criterion))
difference = self.prev_criterion - min_crit
logger.debug("Difference %f" % difference)
try:
max_iteration = int(getval(run_settings, '%s/input/hrmc/max_iteration' % RMIT_SCHEMA))
except (ValueError, SettingNotFoundException):
raise BadInputException("unknown max_iteration")
logger.debug("max_iteration=%s" % max_iteration)
try:
self.error_threshold = float(getval(run_settings, '%s/input/hrmc/error_threshold' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
raise BadInputException("uknown error threshold")
logger.debug("error_threshold=%s" % self.error_threshold)
if self.id >= (max_iteration - 1):
logger.debug("Max Iteration Reached %d " % self.id)
return (True, min_crit)
elif min_crit <= self.prev_criterion and difference <= self.error_threshold:
logger.debug("Convergence reached %f" % difference)
return (True, min_crit)
else:
if difference < 0:
logger.debug("iteration diverged")
logger.debug("iteration continues: %d iteration so far" % self.id)
return (False, min_crit)
def process_outputs(self, run_settings, base_dir, output_url, all_settings,
offset):
# output_dir = 118.138.241.232/outptuersdfsd/sweep277/hrmc278/output_1
# output_prefix = ssh://unix@
# node_output_dir = 2
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
iter_output_dir = os.path.join(os.path.join(base_dir,
"output_%s" % id))
logger.debug('iter_output_dir=%s' % iter_output_dir)
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
logger.debug('output_prefix=%s' % output_prefix)
#iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
logger.debug('output_url=%s' % output_url)
(scheme, host, iter_output_path, location,
query_settings) = storage.parse_bdpurl(output_url)
logger.debug("iter_output_path=%s" % iter_output_path)
iter_out_fsys = storage.get_filesystem(output_url)
logger.debug("iter_out_fsys=%s" % iter_out_fsys)
node_output_dirnames, _ = iter_out_fsys.listdir(iter_output_path)
logger.debug('node_output_dirnames=%s' % node_output_dirnames)
self.audit = ""
Node_info = namedtuple('Node_info', ['dirname', 'number', 'criterion'])
BASE_FNAME = "HRMC.inp"
# generate criterias
self.outputs = []
for node_output_dirname in node_output_dirnames:
node_path = output_prefix + os.path.join(iter_output_dir,
node_output_dirname)
criterion = self.compute_psd_criterion(all_settings, node_path)
#criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dirname, fs,)
logger.debug("criterion=%s" % criterion)
try:
values_url = get_url_with_credentials(
all_settings,
os.path.join(node_path, '%s_values' % BASE_FNAME),
is_relative_path=False)
values_content = storage.get_file(values_url)
logger.debug("values_file=%s" % values_url)
except IOError:
logger.warn("no values file found")
values_map = {}
else:
values_map = dict(json.loads(values_content))
self.outputs.append(
Node_info(dirname=node_output_dirname,
number=values_map['run_counter'],
criterion=criterion))
if not self.outputs:
logger.error("no ouput found for this iteration")
return
self.outputs.sort(key=lambda x: int(x.criterion))
logger.debug("self.outputs=%s" % self.outputs)
try:
# FIXME: need to validate this output to make sure list of int
threshold = ast.literal_eval(
getval(run_settings, '%s/input/hrmc/threshold' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
logger.warn("no threshold found when expected")
return False
logger.debug("threshold = %s" % threshold)
total_picks = 1
if len(threshold) > 1:
for i in threshold:
total_picks *= threshold[i]
else:
total_picks = threshold[0]
def copy_files_with_pattern(iter_out_fsys, source_path, dest_path,
pattern, all_settings):
"""
"""
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
logger.debug('source_path=%s, dest_path=%s' %
(source_path, dest_path))
# (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(source_path)
_, node_output_fnames = iter_out_fsys.listdir(source_path)
ip_address = all_settings['ip_address']
for f in node_output_fnames:
if fnmatch.fnmatch(f, pattern):
source_url = get_url_with_credentials(
all_settings,
output_prefix +
os.path.join(ip_address, source_path, f),
is_relative_path=False)
dest_url = get_url_with_credentials(
all_settings,
output_prefix + os.path.join(ip_address, dest_path, f),
is_relative_path=False)
logger.debug('source_url=%s, dest_url=%s' %
(source_url, dest_url))
content = storage.get_file(source_url)
storage.put_file(dest_url, content)
# Make new input dirs
new_input_dir = os.path.join(
os.path.join(base_dir, "input_%d" % (id + 1)))
for index in range(0, total_picks):
Node_info = self.outputs[index]
logger.debug("node_info.dirname=%s" % Node_info.dirname)
logger.debug("Node_info=%s" % str(Node_info))
new_input_path = os.path.join(new_input_dir, Node_info.dirname)
logger.debug("New input node dir %s" % new_input_path)
old_output_path = os.path.join(iter_output_dir, Node_info.dirname)
# Move all existing domain input files unchanged to next input directory
for f in DOMAIN_INPUT_FILES:
source_url = get_url_with_credentials(
all_settings,
output_prefix + os.path.join(old_output_path, f),
is_relative_path=False)
dest_url = get_url_with_credentials(
all_settings,
output_prefix + os.path.join(new_input_path, f),
is_relative_path=False)
logger.debug('source_url=%s, dest_url=%s' %
(source_url, dest_url))
content = storage.get_file(source_url)
logger.debug('content collected')
storage.put_file(dest_url, content)
logger.debug('put successfully')
logger.debug('put file successfully')
pattern = "*_values"
output_offset = os.path.join(
os.path.join(offset, "output_%s" % id, Node_info.dirname))
input_offset = os.path.join(
os.path.join(offset, "input_%s" % (id + 1), Node_info.dirname))
copy_files_with_pattern(iter_out_fsys, output_offset, input_offset,
pattern, all_settings)
pattern = "*_template"
copy_files_with_pattern(iter_out_fsys, output_offset, input_offset,
pattern, all_settings)
# NB: Converge stage triggers based on criterion value from audit.
logger.debug('starting audit')
info = "Run %s preserved (error %s)\n" % (Node_info.number,
Node_info.criterion)
audit_url = get_url_with_credentials(
all_settings,
output_prefix + os.path.join(new_input_path, 'audit.txt'),
is_relative_path=False)
storage.put_file(audit_url, info)
logger.debug("audit=%s" % info)
logger.debug('1:audit_url=%s' % audit_url)
self.audit += info
# move xyz_final.xyz to initial.xyz
source_url = get_url_with_credentials(
all_settings,
output_prefix + os.path.join(old_output_path, "xyz_final.xyz"),
is_relative_path=False)
logger.debug('source_url=%s' % source_url)
dest_url = get_url_with_credentials(
all_settings,
output_prefix +
os.path.join(new_input_path, 'input_initial.xyz'),
is_relative_path=False)
logger.debug('dest_url=%s' % dest_url)
content = storage.get_file(source_url)
logger.debug('content=%s' % content)
storage.put_file(dest_url, content)
self.audit += "spawning diamond runs\n"
logger.debug(
"input_dir=%s" %
(output_prefix + os.path.join(new_input_dir, 'audit.txt')))
audit_url = get_url_with_credentials(
all_settings,
output_prefix + os.path.join(new_input_dir, 'audit.txt'),
is_relative_path=False)
logger.debug('audit_url=%s' % audit_url)
storage.put_file(audit_url, self.audit)
def curate_dataset(self, run_settings, experiment_id, base_dir, output_url,
all_settings):
iteration = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
iter_output_dir = os.path.join(
os.path.join(base_dir, "output_%s" % iteration))
output_prefix = '%s://%s@' % (all_settings['scheme'],
all_settings['type'])
iter_output_dir = "%s%s" % (output_prefix, iter_output_dir)
(scheme, host, mypath, location,
query_settings) = storage.parse_bdpurl(output_url)
fsys = storage.get_filesystem(output_url)
node_output_dirnames, _ = fsys.listdir(mypath)
logger.debug("node_output_dirnames=%s" % node_output_dirnames)
if all_settings['mytardis_host']:
for i, node_output_dirname in enumerate(node_output_dirnames):
node_path = os.path.join(iter_output_dir, node_output_dirname)
# find criterion
crit = None # is there an infinity criterion
for ni in self.outputs:
if ni.dirname == node_output_dirname:
crit = ni.criterion
break
else:
logger.debug("criterion not found")
continue
logger.debug("crit=%s" % crit)
# graph_params = []
def extract_psd_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
res = {"hrmcdfile/r1": xs, "hrmcdfile/g1": ys}
return res
def extract_psdexp_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
res = {"hrmcdfile/r2": xs, "hrmcdfile/g2": ys}
return res
def extract_grfinal_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
#FIXME: len(xs) == len(ys) for this to work.
#TODO: hack to handle when xs and ys are too
# large to fit in Parameter with db_index.
# solved by function call at destination
cut_xs = [
xs[i] for i, x in enumerate(xs)
if (i % (len(xs) / 20) == 0)
]
cut_ys = [
ys[i] for i, x in enumerate(ys)
if (i % (len(ys) / 20) == 0)
]
res = {"hrmcdfile/r3": cut_xs, "hrmcdfile/g3": cut_ys}
return res
def extract_inputgr_func(fp):
res = []
xs = []
ys = []
for i, line in enumerate(fp):
columns = line.split()
xs.append(float(columns[0]))
ys.append(float(columns[1]))
#FIXME: len(xs) == len(ys) for this to work.
#TODO: hack to handle when xs and ys are too
# large to fit in Parameter with db_index.
# solved by function call at destination
cut_xs = [
xs[i] for i, x in enumerate(xs)
if (i % (len(xs) / 20) == 0)
]
cut_ys = [
ys[i] for i, x in enumerate(ys)
if (i % (len(ys) / 20) == 0)
]
res = {"hrmcdfile/r4": cut_xs, "hrmcdfile/g4": cut_ys}
return res
#TODO: hrmcexp graph should be tagged to input directories (not output directories)
#because we want the result after pruning.
#todo: replace self.boto_setttings with mytardis_settings
EXP_DATASET_NAME_SPLIT = 2
def get_exp_name_for_output(settings, url, path):
# return str(os.sep.join(path.split(os.sep)[:-EXP_DATASET_NAME_SPLIT]))
return str(os.sep.join(path.split(os.sep)[-4:-2]))
def get_dataset_name_for_output(settings, url, path):
logger.debug("path=%s" % path)
host = settings['host']
prefix = 'ssh://%s@%s' % (settings['type'], host)
source_url = get_url_with_credentials(
settings,
os.path.join(prefix, path, "HRMC.inp_values"),
is_relative_path=False)
logger.debug("source_url=%s" % source_url)
try:
content = storage.get_file(source_url)
except IOError, e:
logger.warn("cannot read file %s" % e)
return str(
os.sep.join(
path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
logger.debug("content=%s" % content)
try:
values_map = dict(json.loads(str(content)))
except Exception, e:
logger.warn("cannot load %s: %s" % (content, e))
return str(
os.sep.join(
path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
try:
iteration = str(path.split(os.sep)[-2:-1][0])
except Exception, e:
logger.error(e)
iteration = ""
if "_" in iteration:
iteration = iteration.split("_")[1]
else:
iteration = "final"
dataset_name = "%s_%s_%s" % (
iteration, values_map['generator_counter'],
values_map['run_counter'])
logger.debug("dataset_name=%s" % dataset_name)
return dataset_name