def _upload_variations_inputs(settings, source_url_initial, values_map): bdp_username = settings['bdp_username'] logger.debug("source_url_initial=%s" % source_url_initial) encoded_s_url = storage.get_url_with_credentials(settings, source_url_initial) logger.debug("encoded_s_url=%s" % encoded_s_url) dest_url = _get_dest_bdp_url(settings) computation_platform_url = settings['comp_platform_url'] bdp_username = settings['bdp_username'] comp_pltf_settings = manage.get_platform_settings( computation_platform_url, bdp_username) settings.update(comp_pltf_settings) encoded_d_url = storage.get_url_with_credentials(settings, dest_url, is_relative_path=True, ip_address=settings['host']) storage.copy_directories(encoded_s_url, encoded_d_url) for content_fname, content in _instantiate_context( source_url_initial, settings, values_map).items(): content_url = storage.get_url_with_credentials( settings, os.path.join(dest_url, content_fname), is_relative_path=True, ip_address=settings['host']) logger.debug("content_url=%s" % content_url) storage.put_file(content_url, content.encode('utf-8')) _save_values(settings, dest_url, values_map) logger.debug("done input upload")
def copy_files_with_pattern(iter_out_fsys, source_path, dest_path, pattern, all_settings): """ """ output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('source_path=%s, dest_path=%s' % (source_path, dest_path)) # (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(source_path) _, node_output_fnames = iter_out_fsys.listdir(source_path) ip_address = all_settings['ip_address'] for f in node_output_fnames: if fnmatch.fnmatch(f, pattern): source_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(ip_address, source_path, f), is_relative_path=False) dest_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(ip_address, dest_path, f), is_relative_path=False) logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) content = storage.get_file(source_url) storage.put_file(dest_url, content)
def put_proc_ids(relative_path, ids, ip, settings): relative_path = os.path.join(relative_path, settings['filename_for_PIDs']) logger.debug('put_proc_ids=%s' % relative_path) destination = get_url_with_credentials(settings, relative_path, is_relative_path=True, ip_address=ip) logger.debug('destination=%s' % destination) ids_str = [] [ids_str.append(str(i)) for i in ids] proc_ids = ("\n".join(ids_str)) + "\n" logger.debug('ids_str=%s' % ids_str) logger.debug('proc_ids=%s' % proc_ids) logger.debug('encoded=%s' % proc_ids.encode('utf-8')) put_file(destination, proc_ids.encode('utf-8'))
def put_proc_ids(relative_path, ids, ip, settings): relative_path = os.path.join(relative_path, settings['filename_for_PIDs']) logger.debug('put_proc_ids=%s' % relative_path) destination = get_url_with_credentials(settings, relative_path, is_relative_path=True, ip_address=ip) logger.debug('destination=%s' % destination) ids_str = [] [ids_str.append(str(i)) for i in ids] proc_ids = ("\n".join(ids_str)) + "\n" logger.debug('ids_str=%s' % ids_str) logger.debug('proc_ids=%s' % proc_ids) logger.debug('encoded=%s' % proc_ids.encode('utf-8')) put_file(destination, proc_ids.encode('utf-8'))
def copy_files_with_pattern(iter_out_fsys, source_path, dest_path, pattern, all_settings): """ """ output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('source_path=%s, dest_path=%s' % (source_path, dest_path)) # (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(source_path) _, node_output_fnames = iter_out_fsys.listdir(source_path) ip_address = all_settings['ip_address'] for f in node_output_fnames: if fnmatch.fnmatch(f, pattern): source_url = get_url_with_credentials(all_settings, output_prefix + os.path.join(ip_address, source_path, f), is_relative_path=False) dest_url = get_url_with_credentials(all_settings, output_prefix + os.path.join(ip_address, dest_path, f), is_relative_path=False) logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) content = storage.get_file(source_url) storage.put_file(dest_url, content)
def put_dest_file(proc, fname, dest_file_location, resched_file_location, content): dest_url = get_url_with_credentials( computation_platform_settings, os.path.join(dest_file_location, fname), is_relative_path=True, ip_address=proc['ip_address']) logger.debug("writing to =%s" % dest_url) #logger.debug("content=%s" % content) storage.put_file(dest_url, content) if self.reschedule_failed_procs: logger.debug("resched=%s" % resched_file_location) logger.debug("fname=%s" % fname) logger.debug("output_storage_settings=%s" % output_storage_settings) logger.debug("here") test = "%s/%s" % (resched_file_location, fname) logger.debug("test=%s" % test) resched_url = get_url_with_credentials( output_storage_settings, test) logger.debug("writing backup to %s" % resched_url) storage.put_file(resched_url, content) logger.debug("done")
def put_dest_file(proc, fname, dest_file_location, resched_file_location, content): dest_url = get_url_with_credentials( computation_platform_settings, os.path.join(dest_file_location, fname), is_relative_path=True, ip_address=proc['ip_address']) logger.debug("writing to =%s" % dest_url) #logger.debug("content=%s" % content) storage.put_file(dest_url, content) if self.reschedule_failed_procs: logger.debug("resched=%s" % resched_file_location) logger.debug("fname=%s" % fname) logger.debug("output_storage_settings=%s" % output_storage_settings) logger.debug("here") test = "%s/%s" % (resched_file_location, fname) logger.debug("test=%s" % test) resched_url = get_url_with_credentials( output_storage_settings, test) logger.debug("writing backup to %s" % resched_url) storage.put_file(resched_url, content) logger.debug("done")
logger.debug("values_source_url=%s" % values_source_url) values_dest_url = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.job_dir, self.output_dir, process_id, django_settings.VALUES_FNAME), is_relative_path=False) logger.debug("values_dest_url=%s" % values_dest_url) try: logger.debug('reading %s' % values_source_url) content = storage.get_file(values_source_url) except IOError, e: content = {} logger.debug('content=%s' % content) storage.put_file(values_dest_url, content) def process(self, run_settings): """ Check all registered nodes to find whether they are running, stopped or in error_nodes """ local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS) # local_settings = run_settings[models.UserProfile.PROFILE_SCHEMA_NS] retrieve_local_settings(run_settings, local_settings) logger.debug("local_settings=%s" % local_settings) self.contextid = getval( run_settings,
def _upload_input_dir_variations(self, processes, local_settings, computation_platform_settings, output_storage_settings, mytardis_settings, input_dir, run_settings): output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) input_url_with_credentials = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join( self.iter_inputdir, input_dir), is_relative_path=False) logger.debug('input_url_with_credentials=%s' % input_url_with_credentials) if local_settings['curate_data']: try: mytardis_platform = jobs.safe_import('chiminey.platform.mytardis.MyTardisPlatform', [], {}) self.experiment_id = mytardis_platform.create_dataset_for_input(self.experiment_id, run_settings, local_settings, output_storage_settings, mytardis_settings, input_url_with_credentials) except ImproperlyConfigured as e: logger.error("Cannot load mytardis platform hook %s" % e) else: logger.warn('Data curation is off') # get run Map parent_stage = self.import_parent_stage(run_settings) run_map, self.rand_index = parent_stage.get_internal_sweep_map(local_settings, run_settings=run_settings) # load value_map values_url_with_pkey = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.iter_inputdir, input_dir, self.VALUES_FNAME), is_relative_path=False) logger.debug("initial values_file=%s" % values_url_with_pkey) values = {} try: values_content = storage.get_file(values_url_with_pkey) except IOError: logger.warn("no values file found") else: logger.debug("values_content = %s" % values_content) values = dict(json.loads(values_content)) logger.debug("values=%s" % values) # generates a set of variations for the template fname logger.debug('self.initial_numbfile = %s ' % self.initial_numbfile) contexts = self._get_variation_contexts( [run_map], values, self.initial_numbfile) self.initial_numbfile += len(contexts) logger.debug('contexts = %s ' % contexts) logger.debug('self.initial_numbfile = %s ' % self.initial_numbfile) # for each context, copy each file to dest and any # templates to be instantiated, then store in values. template_pat = re.compile("(.*)_template") relative_path_suffix = self.get_relative_output_path(local_settings) for context in contexts: logger.debug("context=%s" % context) # get list of all files in input_dir fname_url_with_pkey = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.iter_inputdir, input_dir), is_relative_path=False) input_files = storage.list_dirs(fname_url_with_pkey, list_files=True) # get process information run_counter = context['run_counter'] logger.debug("run_counter=%s" % run_counter) proc = None for p in processes: # TODO: how to handle invalid run_counter pid = int(p['id']) logger.debug("pid=%s" % pid) if pid == run_counter: proc = p break else: logger.error("no process found matching run_counter") raise BadInputException() logger.debug("proc=%s" % pformat(proc)) for fname in input_files: logger.debug("fname=%s" % fname) templ_mat = template_pat.match(fname) fname_url_with_credentials = storage.get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.iter_inputdir, input_dir, fname), is_relative_path=False) logger.debug("fname_url_with_credentials=%s" % fname_url_with_credentials) def put_dest_file(proc, fname, dest_file_location, resched_file_location, content): dest_url = get_url_with_credentials( computation_platform_settings, os.path.join( dest_file_location, fname), is_relative_path=True, ip_address=proc['ip_address']) logger.debug("writing to =%s" % dest_url) #logger.debug("content=%s" % content) storage.put_file(dest_url, content) if self.reschedule_failed_procs: logger.debug("resched=%s" % resched_file_location) logger.debug("fname=%s" % fname) logger.debug("output_storage_settings=%s" % output_storage_settings) logger.debug("here") test = "%s/%s" % (resched_file_location, fname) logger.debug("test=%s" % test) resched_url = get_url_with_credentials( output_storage_settings, test) logger.debug("writing backup to %s" % resched_url) storage.put_file(resched_url, content) logger.debug("done") outputs = [] if templ_mat: base_fname = templ_mat.group(1) template_content = storage.get_file( fname_url_with_credentials) try: templ = Template(template_content) except TemplateSyntaxError, e: logger.error(e) # FIXME: should detect this during submission of job, # as no sensible way to recover here. # TODO: signal error conditions in job status continue new_context = Context(context) logger.debug("new_content=%s" % new_context) render_output = templ.render(new_context) render_output = render_output.encode('utf-8') outputs.append((base_fname, render_output)) outputs.append((fname, template_content)) else: content = storage.get_file(fname_url_with_credentials) outputs.append((fname, content)) for (new_fname, content) in outputs: dest_file_location = computation_platform_settings['type']\ + "@" + os.path.join(relative_path_suffix, proc['id'], local_settings['smart_connector_input']) logger.debug("dest_file_location =%s" % dest_file_location) resched_file_location = "%s%s" % (output_prefix, os.path.join( self.job_dir, "input_backup", proc['id'])) logger.debug("resched_file_location=%s" % resched_file_location) put_dest_file(proc, new_fname, dest_file_location, resched_file_location, content) # then copy context new values file logger.debug("writing values file") values_dest_location = computation_platform_settings['type']\ + "@" + os.path.join(relative_path_suffix, proc['id'], local_settings['smart_connector_input'], self.VALUES_FNAME) logger.debug("values_dest_location =%s" % values_dest_location) values_dest_url = get_url_with_credentials( computation_platform_settings, values_dest_location, is_relative_path=True, ip_address=proc['ip_address']) storage.put_file(values_dest_url, json.dumps(context, indent=4))
def compute_psd_criterion(self, all_settings, node_path): import math import os #globalFileSystem = fs.get_global_filesystem() # psd = os.path.join(globalFileSystem, # self.output_dir, node_output_dir, # "PSD_output/psd.dat") #Fixme replace all reference to files by parameters, e.g PSDCode output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('output_prefix=%s' % output_prefix) logger.debug('node_path=%s' % node_path) logger.debug('compute psd---') psd_url = get_url_with_credentials(all_settings, os.path.join( node_path, "PSD_output", "psd.dat"), is_relative_path=False) logger.debug('psd_url=%s' % psd_url) psd = storage.get_filep(psd_url) logger.debug('psd=%s' % psd._name) # psd_exp = os.path.join(globalFileSystem, # self.output_dir, node_output_dir, # "PSD_output/PSD_exp.dat") psd_url = get_url_with_credentials(all_settings, os.path.join( node_path, "PSD_output", "PSD_exp.dat"), is_relative_path=False) logger.debug('psd_url=%s' % psd_url) psd_exp = storage.get_filep(psd_url) logger.debug('psd_exp=%s' % psd_exp._name) logger.debug("PSD %s %s " % (psd._name, psd_exp._name)) x_axis = [] y1_axis = [] for line in psd: column = line.split() #logger.debug(column) if len(column) > 0: x_axis.append(float(column[0])) y1_axis.append(float(column[1])) logger.debug("x_axis \n %s" % x_axis) logger.debug("y1_axis \n %s" % y1_axis) y2_axis = [] for line in psd_exp: column = line.split() #logger.debug(column) if len(column) > 0: y2_axis.append(float(column[1])) for i in range(len(x_axis) - len(y2_axis)): y2_axis.append(0) logger.debug("y2_axis \n %s" % y2_axis) criterion = 0 for i in range(len(y1_axis)): criterion += math.pow((y1_axis[i] - y2_axis[i]), 2) logger.debug("Criterion %f" % criterion) criterion_url = get_url_with_credentials(all_settings, os.path.join( node_path, "PSD_output", "criterion.txt"), is_relative_path=False) storage.put_file(criterion_url, str(criterion)) return criterion
def process_outputs(self, run_settings, base_dir, output_url, all_settings, offset): # output_dir = 118.138.241.232/outptuersdfsd/sweep277/hrmc278/output_1 # output_prefix = ssh://unix@ # node_output_dir = 2 output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA)) iter_output_dir = os.path.join(os.path.join(base_dir, "output_%s" % id)) logger.debug('iter_output_dir=%s' % iter_output_dir) output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('output_prefix=%s' % output_prefix) #iter_output_dir = "%s%s" % (output_prefix, iter_output_dir) logger.debug('output_url=%s' % output_url) (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(output_url) logger.debug("iter_output_path=%s" % iter_output_path) iter_out_fsys = storage.get_filesystem(output_url) logger.debug("iter_out_fsys=%s" % iter_out_fsys) node_output_dirnames, _ = iter_out_fsys.listdir(iter_output_path) logger.debug('node_output_dirnames=%s' % node_output_dirnames) self.audit = "" Node_info = namedtuple('Node_info', ['dirname', 'number', 'criterion']) BASE_FNAME = "HRMC.inp" # generate criterias self.outputs = [] for node_output_dirname in node_output_dirnames: node_path = output_prefix + os.path.join(iter_output_dir, node_output_dirname) criterion = self.compute_psd_criterion(all_settings, node_path) #criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dirname, fs,) logger.debug("criterion=%s" % criterion) try: values_url = get_url_with_credentials( all_settings, os.path.join(node_path, '%s_values' % BASE_FNAME), is_relative_path=False) values_content = storage.get_file(values_url) logger.debug("values_file=%s" % values_url) except IOError: logger.warn("no values file found") values_map = {} else: values_map = dict(json.loads(values_content)) self.outputs.append(Node_info(dirname=node_output_dirname, number=values_map['run_counter'], criterion=criterion)) if not self.outputs: logger.error("no ouput found for this iteration") return self.outputs.sort(key=lambda x: int(x.criterion)) logger.debug("self.outputs=%s" % self.outputs) try: # FIXME: need to validate this output to make sure list of int threshold = ast.literal_eval(getval(run_settings, '%s/input/hrmc/threshold' % RMIT_SCHEMA)) except (SettingNotFoundException, ValueError): logger.warn("no threshold found when expected") return False logger.debug("threshold = %s" % threshold) total_picks = 1 if len(threshold) > 1: for i in threshold: total_picks *= threshold[i] else: total_picks = threshold[0] def copy_files_with_pattern(iter_out_fsys, source_path, dest_path, pattern, all_settings): """ """ output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('source_path=%s, dest_path=%s' % (source_path, dest_path)) # (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(source_path) _, node_output_fnames = iter_out_fsys.listdir(source_path) ip_address = all_settings['ip_address'] for f in node_output_fnames: if fnmatch.fnmatch(f, pattern): source_url = get_url_with_credentials(all_settings, output_prefix + os.path.join(ip_address, source_path, f), is_relative_path=False) dest_url = get_url_with_credentials(all_settings, output_prefix + os.path.join(ip_address, dest_path, f), is_relative_path=False) logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) content = storage.get_file(source_url) storage.put_file(dest_url, content) # Make new input dirs new_input_dir = os.path.join(os.path.join(base_dir, "input_%d" % (id + 1))) for index in range(0, total_picks): Node_info = self.outputs[index] logger.debug("node_info.dirname=%s" % Node_info.dirname) logger.debug("Node_info=%s" % str(Node_info)) new_input_path = os.path.join(new_input_dir, Node_info.dirname) logger.debug("New input node dir %s" % new_input_path) old_output_path = os.path.join(iter_output_dir, Node_info.dirname) # Move all existing domain input files unchanged to next input directory for f in DOMAIN_INPUT_FILES: source_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(old_output_path, f), is_relative_path=False) dest_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_path, f), is_relative_path=False) logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) content = storage.get_file(source_url) logger.debug('content collected') storage.put_file(dest_url, content) logger.debug('put successfully') logger.debug('put file successfully') pattern = "*_values" output_offset = os.path.join(os.path.join(offset, "output_%s" % id, Node_info.dirname)) input_offset = os.path.join(os.path.join(offset, "input_%s" % (id + 1), Node_info.dirname)) copy_files_with_pattern(iter_out_fsys, output_offset, input_offset, pattern, all_settings) pattern = "*_template" copy_files_with_pattern(iter_out_fsys, output_offset, input_offset, pattern, all_settings) # NB: Converge stage triggers based on criterion value from audit. logger.debug('starting audit') info = "Run %s preserved (error %s)\n" % (Node_info.number, Node_info.criterion) audit_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_path, 'audit.txt'), is_relative_path=False) storage.put_file(audit_url, info) logger.debug("audit=%s" % info) logger.debug('1:audit_url=%s' % audit_url) self.audit += info # move xyz_final.xyz to initial.xyz source_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(old_output_path, "xyz_final.xyz"), is_relative_path=False) logger.debug('source_url=%s' % source_url) dest_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_path, 'input_initial.xyz'), is_relative_path=False) logger.debug('dest_url=%s' % dest_url) content = storage.get_file(source_url) logger.debug('content=%s' % content) storage.put_file(dest_url, content) self.audit += "spawning diamond runs\n" logger.debug("input_dir=%s" % (output_prefix + os.path.join(new_input_dir, 'audit.txt'))) audit_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_dir, 'audit.txt'), is_relative_path=False) logger.debug('audit_url=%s' % audit_url) storage.put_file(audit_url, self.audit)
def upload_variation_inputs(self, run_settings, local_settings, variations, processes, input_dir, output_storage_settings, computation_platform_settings, mytardis_settings): ''' Create input packages for each variation and upload the vms ''' logger.debug("upload_variation_inputs") output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) source_files_url = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join( self.iter_inputdir, input_dir), is_relative_path=False) logger.debug('source_files_url=%s' % source_files_url) # Copy input directory to mytardis only after saving locally, so if # something goes wrong we still have the results if local_settings['curate_data']: self.experiment_id = self.curate_data(self.experiment_id, local_settings, output_storage_settings, mytardis_settings, source_files_url) else: logger.warn('Data curation is off') #proc_ind = 0 for var_fname in variations.keys(): logger.debug("var_fname=%s" % var_fname) logger.debug('variations[var_fname]=%s' % variations[var_fname]) for var_content, values in variations[var_fname]: #logger.debug("var_content = %s" % var_content) #logger.debug('proc_ind=%s' % proc_ind) logger.debug('processes=%s' % processes) run_counter = values['run_counter'] logger.debug("run_counter=%s" % run_counter) proc = None for p in processes: # TODO: how to handle invalid run_counter pid = int(p['id']) logger.debug("pid=%s" % pid) if pid == run_counter: proc = p break else: logger.error("no process found matching run_counter") #smartconnectorscheduler.error(run_settings, "%s: wait" % (self.id + 1)) # TODO: catch this error and recover raise BadInputException() logger.debug("proc=%s" % pformat(proc)) #proc = processes[proc_ind] #proc_ind += 1 #ip = botocloudconnector.get_instance_ip(var_node.id, local_settings) ip = proc['ip_address'] #dest_files_location = computation_platform_settings['type'] + "@"\ # + os.path.join(local_settings['payload_destination'], # proc['id'], # local_settings['payload_cloud_dirname'] # ) relative_path_suffix = self.get_relative_output_path(local_settings) dest_files_location = computation_platform_settings['type'] + "@"\ + os.path.join(relative_path_suffix, proc['id'], local_settings['payload_cloud_dirname'] ) logger.debug('dest_files_location=%s' % dest_files_location) dest_files_url = get_url_with_credentials( computation_platform_settings, dest_files_location, is_relative_path=True, ip_address=ip) logger.debug('dest_files_url=%s' % dest_files_url) # FIXME: Cleanup any existing runs already there # FIXME: keep the compile exec from setup #FIXME: exceptions should be given as parameter #FIXme we should not delete anyfile. SInce each process runs in its own directory exceptions = [local_settings['compile_file'], "..", ".", 'PSD', 'PSD.f', 'PSD_exp.dat', 'PSD.inp', 'Makefile', 'running.sh', 'process_scheduledone.sh', 'process_schedulestart.sh'] storage.copy_directories(source_files_url, dest_files_url) if self.reschedule_failed_procs: input_backup = os.path.join(self.job_dir, "input_backup", proc['id']) backup_url = get_url_with_credentials( output_storage_settings, output_prefix + input_backup, is_relative_path=False) storage.copy_directories(source_files_url, backup_url) # Why do we need to create a tempory file to make this copy? import uuid randsuffix = unicode(uuid.uuid4()) # should use some job id here var_url = get_url_with_credentials(local_settings, os.path.join("tmp%s" % randsuffix, "var"), is_relative_path=True) logger.debug("var_url=%s" % var_url) storage.put_file(var_url, var_content.encode('utf-8')) value_url = get_url_with_credentials(local_settings, os.path.join("tmp%s" % randsuffix, "value"), is_relative_path=True) logger.debug("value_url=%s" % value_url) storage.put_file(value_url, json.dumps(values)) #local_settings['platform'] should be replaced # and overwrite on the remote #var_fname_remote = computation_platform_settings['type']\ # + "@" + os.path.join(local_settings['payload_destination'], # proc['id'], # local_settings['payload_cloud_dirname'], # var_fname) var_fname_remote = computation_platform_settings['type']\ + "@" + os.path.join(relative_path_suffix, proc['id'], local_settings['payload_cloud_dirname'], var_fname) var_fname_pkey = get_url_with_credentials( computation_platform_settings, var_fname_remote, is_relative_path=True, ip_address=ip) var_content = storage.get_file(var_url) storage.put_file(var_fname_pkey, var_content) logger.debug("var_fname_pkey=%s" % var_fname_pkey) values_fname_pkey = get_url_with_credentials( computation_platform_settings, os.path.join(dest_files_location, "%s_values" % var_fname), is_relative_path=True, ip_address=ip) values_content = storage.get_file(value_url) storage.put_file(values_fname_pkey, values_content) logger.debug("values_fname_pkey=%s" % values_fname_pkey) #copying values and var_content to backup folder if self.reschedule_failed_procs: value_url = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(input_backup, "%s_values" % var_fname), is_relative_path=False) logger.debug("value_url=%s" % value_url) storage.put_file(value_url, json.dumps(values)) var_fname_pkey = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(input_backup, var_fname), is_relative_path=False) var_content = storage.get_file(var_url) storage.put_file(var_fname_pkey, var_content) # cleanup tmp_url = get_url_with_credentials(local_settings, os.path.join("tmp%s" % randsuffix), is_relative_path=True) logger.debug("deleting %s" % tmp_url)
def process_outputs(self, run_settings, base_dir, output_url, all_settings, offset): # output_dir = 118.138.241.232/outptuersdfsd/sweep277/hrmc278/output_1 # output_prefix = ssh://unix@ # node_output_dir = 2 output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA)) iter_output_dir = os.path.join(os.path.join(base_dir, "output_%s" % id)) logger.debug('iter_output_dir=%s' % iter_output_dir) output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('output_prefix=%s' % output_prefix) #iter_output_dir = "%s%s" % (output_prefix, iter_output_dir) logger.debug('output_url=%s' % output_url) (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(output_url) logger.debug("iter_output_path=%s" % iter_output_path) iter_out_fsys = storage.get_filesystem(output_url) logger.debug("iter_out_fsys=%s" % iter_out_fsys) node_output_dirnames, _ = iter_out_fsys.listdir(iter_output_path) logger.debug('node_output_dirnames=%s' % node_output_dirnames) self.audit = "" Node_info = namedtuple('Node_info', ['dirname', 'number', 'criterion']) BASE_FNAME = "HRMC.inp" # generate criterias self.outputs = [] for node_output_dirname in node_output_dirnames: node_path = output_prefix + os.path.join(iter_output_dir, node_output_dirname) criterion = self.compute_psd_criterion(all_settings, node_path) #criterion = self.compute_hrmc_criterion(values_map['run_counter'], node_output_dirname, fs,) logger.debug("criterion=%s" % criterion) try: values_url = get_url_with_credentials( all_settings, os.path.join(node_path, '%s_values' % BASE_FNAME), is_relative_path=False) values_content = storage.get_file(values_url) logger.debug("values_file=%s" % values_url) except IOError: logger.warn("no values file found") values_map = {} else: values_map = dict(json.loads(values_content)) self.outputs.append( Node_info(dirname=node_output_dirname, number=values_map['run_counter'], criterion=criterion)) if not self.outputs: logger.error("no ouput found for this iteration") return self.outputs.sort(key=lambda x: int(x.criterion)) logger.debug("self.outputs=%s" % self.outputs) try: # FIXME: need to validate this output to make sure list of int threshold = ast.literal_eval( getval(run_settings, '%s/input/hrmc/threshold' % RMIT_SCHEMA)) except (SettingNotFoundException, ValueError): logger.warn("no threshold found when expected") return False logger.debug("threshold = %s" % threshold) total_picks = 1 if len(threshold) > 1: for i in threshold: total_picks *= threshold[i] else: total_picks = threshold[0] def copy_files_with_pattern(iter_out_fsys, source_path, dest_path, pattern, all_settings): """ """ output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('source_path=%s, dest_path=%s' % (source_path, dest_path)) # (scheme, host, iter_output_path, location, query_settings) = storage.parse_bdpurl(source_path) _, node_output_fnames = iter_out_fsys.listdir(source_path) ip_address = all_settings['ip_address'] for f in node_output_fnames: if fnmatch.fnmatch(f, pattern): source_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(ip_address, source_path, f), is_relative_path=False) dest_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(ip_address, dest_path, f), is_relative_path=False) logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) content = storage.get_file(source_url) storage.put_file(dest_url, content) # Make new input dirs new_input_dir = os.path.join( os.path.join(base_dir, "input_%d" % (id + 1))) for index in range(0, total_picks): Node_info = self.outputs[index] logger.debug("node_info.dirname=%s" % Node_info.dirname) logger.debug("Node_info=%s" % str(Node_info)) new_input_path = os.path.join(new_input_dir, Node_info.dirname) logger.debug("New input node dir %s" % new_input_path) old_output_path = os.path.join(iter_output_dir, Node_info.dirname) # Move all existing domain input files unchanged to next input directory for f in DOMAIN_INPUT_FILES: source_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(old_output_path, f), is_relative_path=False) dest_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_path, f), is_relative_path=False) logger.debug('source_url=%s, dest_url=%s' % (source_url, dest_url)) content = storage.get_file(source_url) logger.debug('content collected') storage.put_file(dest_url, content) logger.debug('put successfully') logger.debug('put file successfully') pattern = "*_values" output_offset = os.path.join( os.path.join(offset, "output_%s" % id, Node_info.dirname)) input_offset = os.path.join( os.path.join(offset, "input_%s" % (id + 1), Node_info.dirname)) copy_files_with_pattern(iter_out_fsys, output_offset, input_offset, pattern, all_settings) pattern = "*_template" copy_files_with_pattern(iter_out_fsys, output_offset, input_offset, pattern, all_settings) # NB: Converge stage triggers based on criterion value from audit. logger.debug('starting audit') info = "Run %s preserved (error %s)\n" % (Node_info.number, Node_info.criterion) audit_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_path, 'audit.txt'), is_relative_path=False) storage.put_file(audit_url, info) logger.debug("audit=%s" % info) logger.debug('1:audit_url=%s' % audit_url) self.audit += info # move xyz_final.xyz to initial.xyz source_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(old_output_path, "xyz_final.xyz"), is_relative_path=False) logger.debug('source_url=%s' % source_url) dest_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_path, 'input_initial.xyz'), is_relative_path=False) logger.debug('dest_url=%s' % dest_url) content = storage.get_file(source_url) logger.debug('content=%s' % content) storage.put_file(dest_url, content) self.audit += "spawning diamond runs\n" logger.debug( "input_dir=%s" % (output_prefix + os.path.join(new_input_dir, 'audit.txt'))) audit_url = get_url_with_credentials( all_settings, output_prefix + os.path.join(new_input_dir, 'audit.txt'), is_relative_path=False) logger.debug('audit_url=%s' % audit_url) storage.put_file(audit_url, self.audit)
def compute_psd_criterion(self, all_settings, node_path): import math import os #globalFileSystem = fs.get_global_filesystem() # psd = os.path.join(globalFileSystem, # self.output_dir, node_output_dir, # "PSD_output/psd.dat") #Fixme replace all reference to files by parameters, e.g PSDCode output_prefix = '%s://%s@' % (all_settings['scheme'], all_settings['type']) logger.debug('output_prefix=%s' % output_prefix) logger.debug('node_path=%s' % node_path) logger.debug('compute psd---') psd_url = get_url_with_credentials(all_settings, os.path.join(node_path, "PSD_output", "psd.dat"), is_relative_path=False) logger.debug('psd_url=%s' % psd_url) psd = storage.get_filep(psd_url) logger.debug('psd=%s' % psd._name) # psd_exp = os.path.join(globalFileSystem, # self.output_dir, node_output_dir, # "PSD_output/PSD_exp.dat") psd_url = get_url_with_credentials( all_settings, os.path.join(node_path, "PSD_output", "PSD_exp.dat"), is_relative_path=False) logger.debug('psd_url=%s' % psd_url) psd_exp = storage.get_filep(psd_url) logger.debug('psd_exp=%s' % psd_exp._name) logger.debug("PSD %s %s " % (psd._name, psd_exp._name)) x_axis = [] y1_axis = [] for line in psd: column = line.split() #logger.debug(column) if len(column) > 0: x_axis.append(float(column[0])) y1_axis.append(float(column[1])) logger.debug("x_axis \n %s" % x_axis) logger.debug("y1_axis \n %s" % y1_axis) y2_axis = [] for line in psd_exp: column = line.split() #logger.debug(column) if len(column) > 0: y2_axis.append(float(column[1])) for i in range(len(x_axis) - len(y2_axis)): y2_axis.append(0) logger.debug("y2_axis \n %s" % y2_axis) criterion = 0 for i in range(len(y1_axis)): criterion += math.pow((y1_axis[i] - y2_axis[i]), 2) logger.debug("Criterion %f" % criterion) criterion_url = get_url_with_credentials( all_settings, os.path.join(node_path, "PSD_output", "criterion.txt"), is_relative_path=False) storage.put_file(criterion_url, str(criterion)) return criterion
logger.debug("values_source_url=%s" % values_source_url) values_dest_url = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join( self.job_dir, self.output_dir, process_id, django_settings.VALUES_FNAME), is_relative_path=False) logger.debug("values_dest_url=%s" % values_dest_url) try: logger.debug('reading %s' % values_source_url) content = storage.get_file(values_source_url) except IOError, e: content = {} logger.debug('content=%s' % content) storage.put_file(values_dest_url, content) def process(self, run_settings): """ Check all registered nodes to find whether they are running, stopped or in error_nodes """ local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS) # local_settings = run_settings[models.UserProfile.PROFILE_SCHEMA_NS] retrieve_local_settings(run_settings, local_settings) logger.debug("local_settings=%s" % local_settings) self.contextid = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX) output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX) output_storage_settings = get_platform_settings(output_storage_url, local_settings['bdp_username'])
def _save_values(settings, url, context): values_url = storage.get_url_with_credentials(settings, os.path.join(url, VALUES_FNAME), is_relative_path=True, ip_address=settings['host']) storage.put_file(values_url, json.dumps(context))
class Sweep(Stage): def __init__(self, user_settings=None): self.numbfile = 0 logger.debug("Sweep stage initialized") def is_triggered(self, run_settings): logger.debug('run_settings=%s' % run_settings) try: configure_done = int(getval(run_settings, '%s/stages/sweep/sweep_done' % RMIT_SCHEMA)) except (ValueError, SettingNotFoundException): return True return not configure_done def _get_sweep_name(self, run_settings): try: sweep_name = getval(run_settings, '%s/directive_profile/sweep_name' % RMIT_SCHEMA) except SettingNotFoundException: sweep_name = 'unknown_sweep' return sweep_name def process(self, run_settings): logger.debug('run_settings=%s' % run_settings) # Need to make copy because we pass on run_settings to sub connector # so any changes we make here to run_settings WILL be inherited def make_local_settings(run_settings): from copy import deepcopy local_settings = deepcopy(getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)) update(local_settings, run_settings, RMIT_SCHEMA + '/system/platform', # RMIT_SCHEMA + '/input/mytardis/experiment_id', # RMIT_SCHEMA + '/system/random_numbers', ) local_settings['bdp_username'] = getval( run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA) return local_settings local_settings = make_local_settings(run_settings) logger.debug('local_settings=%s' % local_settings) setval(run_settings, '%s/platform/computation/platform_url' % RMIT_SCHEMA, getval(run_settings, '%s/input/system/compplatform/computation_platform' % RMIT_SCHEMA)) def _parse_output_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return name, offset contextid = int(getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)) logger.debug("contextid=%s" % contextid) sweep_name = self._get_sweep_name(run_settings) logger.debug("sweep_name=%s" % sweep_name) output_loc = self.output_exists(run_settings) location = "" if output_loc: location = getval(run_settings, output_loc) output_storage_name, output_storage_offset = \ _parse_output_location(run_settings, location) setval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA, output_storage_name) setval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA, os.path.join(output_storage_offset, '%s%s' % (sweep_name, contextid))) def _parse_input_location(run_settings, location): loc_list = location.split('/') name = loc_list[0] offset = '' if len(loc_list) > 1: offset = os.path.join(*loc_list[1:]) logger.debug('offset=%s' % offset) return (name, offset) input_loc = self.input_exists(run_settings) if input_loc: location = getval(run_settings, input_loc) input_storage_name, input_storage_offset = \ _parse_input_location(run_settings, location) setval(run_settings, '%s/platform/storage/input/platform_url' % RMIT_SCHEMA, input_storage_name) # store offsets setval(run_settings, '%s/platform/storage/input/offset' % RMIT_SCHEMA, input_storage_offset) # TODO: replace with scratch space computation platform space self.scratch_platform = '%s%s%s' % ( manage.get_scratch_platform(), sweep_name, contextid) # mytardis if output_loc: try: self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA)) except KeyError: self.experiment_id = 0 except ValueError: self.experiment_id = 0 try: curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA) except SettingNotFoundException: curate_data = False if curate_data: self.experiment_id = self.curate_data(run_settings, location, self.experiment_id) setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id)) # generate all variations map_text = getval(run_settings, '%s/input/sweep/sweep_map' % RMIT_SCHEMA) # map_text = run_settings[RMIT_SCHEMA + '/input/sweep']['sweep_map'] sweep_map = json.loads(map_text) logger.debug("sweep_map=%s" % pformat(sweep_map)) runs = _expand_variations(maps=[sweep_map], values={}) logger.debug("runs=%s" % runs) # Create random numbers if needed # TODO: move iseed out of hrmc into separate generic schema # to use on any sweepable connector and make this function # completely hrmc independent. rands = [] try: self.rand_index = getval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA) logger.debug("rand_index=%s" % self.rand_index) except SettingNotFoundException: pass else: # prep random seeds for each run based off original iseed # FIXME: inefficient for large random file # TODO, FIXME: this is potentially problematic if different # runs end up overlapping in the random numbers they utilise. # solution is to have separate random files per run or partition # big file up. try: num_url = getval(run_settings, "%s/system/random_numbers" % RMIT_SCHEMA) logger.debug('num_url=%s' % num_url) except SettingNotFoundException: pass else: try: local_settings['random_numbers'] = num_url rands = generate_rands(settings=local_settings, start_range=0, end_range=-1, num_required=len(runs), start_index=self.rand_index) logger.debug("rands=%s" % rands) except Exception, e: logger.debug('error') logger.error(e) raise # load initial values map in the input directory which # contains variable to use for all subdirectives starting_map = {} if input_loc: input_storage_settings = self.get_platform_settings( run_settings, 'http://rmit.edu.au/schemas/platform/storage/input') try: input_prefix = '%s://%s@' % (input_storage_settings['scheme'], input_storage_settings['type']) values_url = get_url_with_credentials( input_storage_settings, input_prefix + os.path.join(input_storage_settings['ip_address'], input_storage_offset, "initial", VALUES_MAP_FILE), is_relative_path=False) logger.debug("values_url=%s" % values_url) values_e_url = get_url_with_credentials( local_settings, values_url, is_relative_path=False) logger.debug("values_url=%s" % values_e_url) values_content = get_file(values_e_url) logger.debug("values_content=%s" % values_content) starting_map = dict(json.loads(values_content)) except IOError: logger.warn("no starting values file found") except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass logger.debug("starting_map after initial values=%s" % pformat(starting_map)) # Copy form input values info starting map # FIXME: could have name collisions between form inputs and # starting values. for ns in run_settings: if ns.startswith(RMIT_SCHEMA + "/input"): # for k, v in run_settings[ns].items(): for k, v in getvals(run_settings, ns).items(): starting_map[k] = v logger.debug("starting_map after form=%s" % pformat(starting_map)) # FIXME: we assume we will always have input directory # Get input_url directory input_url = "" if input_loc: input_prefix = '%s://%s@' % (input_storage_settings['scheme'], input_storage_settings['type']) input_url = get_url_with_credentials(input_storage_settings, input_prefix + os.path.join(input_storage_settings['ip_address'], input_storage_offset), is_relative_path=False) logger.debug("input_url=%s" % input_url) current_context = models.Context.objects.get(id=contextid) user = current_context.owner.user.username # For each of the generated runs, copy across initial input # to individual input directories with variation values, # and then schedule subrun of sub directive logger.debug("run_settings=%s" % run_settings) for i, context in enumerate(runs): run_counter = int(context['run_counter']) logger.debug("run_counter=%s" % run_counter) run_inputdir = os.path.join(self.scratch_platform, SUBDIRECTIVE_DIR % {'run_counter': str(run_counter)}, FIRST_ITERATION_DIR,) logger.debug("run_inputdir=%s" % run_inputdir) run_iter_url = get_url_with_credentials(local_settings, run_inputdir, is_relative_path=False) logger.debug("run_iter_url=%s" % run_iter_url) # Duplicate any input_directory into runX duplicates if input_loc: logger.debug("context=%s" % context) logger.debug("systemsettings=%s" % pformat(getvals(run_settings, RMIT_SCHEMA + '/input/system'))) copy_directories(input_url, run_iter_url) # Need to load up existing values, because original input_dir could # have contained values for the whole run # This code is deprecated in favour of single values file. self.error_detected = False try: template_name = getval(run_settings, '%s/stages/sweep/template_name' % RMIT_SCHEMA) except SettingNotFoundException: pass else: logger.debug("template_name=%s" % template_name) v_map = {} try: values_url = get_url_with_credentials( local_settings, os.path.join(run_inputdir, "initial", VALUES_MAP_TEMPLATE_FILE % {'template_name': template_name}), is_relative_path=False) logger.debug("values_url=%s" % values_url) values_content = get_file(values_url) logger.debug("values_content=%s" % values_content) v_map = dict(json.loads(values_content), indent=4) except IOError: logger.warn("no values file found") except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass v_map.update(starting_map) v_map.update(context) logger.debug("new v_map=%s" % v_map) put_file(values_url, json.dumps(v_map, indent=4)) v_map = {} try: values_url = get_url_with_credentials( local_settings, os.path.join(run_inputdir, "initial", VALUES_MAP_FILE), is_relative_path=False) logger.debug("values_url=%s" % values_url) values_content = get_file(values_url) logger.debug("values_content=%s" % values_content) v_map = dict(json.loads(values_content), ) except IOError: logger.warn("no values file found") except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass v_map.update(starting_map) v_map.update(context) logger.debug("new v_map=%s" % v_map) put_file(values_url, json.dumps(v_map, indent=4)) # Set random numbers for subdirective logger.debug("run_settings=%s" % pformat(run_settings)) if rands: setval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA, rands[i]) if input_loc: # Set revised input_location for subdirective setval(run_settings, input_loc, "%s/%s/%s" % (self.scratch_platform, SUBDIRECTIVE_DIR % {'run_counter': str(run_counter)}, FIRST_ITERATION_DIR)) # Redirect input run_input_storage_name, run_input_storage_offset = \ _parse_input_location(run_settings, "local/sweep%s/run%s/input_0" % (contextid, run_counter)) # setval(run_settings, # '%s/platform/storage/input/platform_url' % RMIT_SCHEMA, # run_input_storage_name) # setval(run_settings, # '%s/platform/storage/input/offset' % RMIT_SCHEMA, # run_input_storage_offset) logger.debug("run_settings=%s" % pformat(run_settings)) try: _submit_subdirective("nectar", run_settings, user, current_context) except Exception, e: logger.error(e) raise e
def _upload_input_dir_variations(self, processes, local_settings, computation_platform_settings, output_storage_settings, mytardis_settings, input_dir, run_settings): output_prefix = '%s://%s@' % (output_storage_settings['scheme'], output_storage_settings['type']) input_url_with_credentials = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.iter_inputdir, input_dir), is_relative_path=False) logger.debug('input_url_with_credentials=%s' % input_url_with_credentials) if local_settings['curate_data']: self.experiment_id = self.curate_data(self.experiment_id, local_settings, output_storage_settings, mytardis_settings, input_url_with_credentials) else: logger.warn('Data curation is off') # get run Map parent_stage = self.import_parent_stage(run_settings) run_map, self.rand_index = parent_stage.get_run_map( local_settings, run_settings=run_settings) # load value_map values_url_with_pkey = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.iter_inputdir, input_dir, self.VALUES_FNAME), is_relative_path=False) logger.debug("initial values_file=%s" % values_url_with_pkey) values = {} try: values_content = storage.get_file(values_url_with_pkey) except IOError: logger.warn("no values file found") else: logger.debug("values_content = %s" % values_content) values = dict(json.loads(values_content)) logger.debug("values=%s" % values) # generates a set of variations for the template fname contexts = self._get_variation_contexts([run_map], values, self.initial_numbfile) self.initial_numbfile += len(contexts) # for each context, copy each file to dest and any # templates to be instantiated, then store in values. template_pat = re.compile("(.*)_template") relative_path_suffix = self.get_relative_output_path(local_settings) for context in contexts: logger.debug("context=%s" % context) # get list of all files in input_dir fname_url_with_pkey = get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.iter_inputdir, input_dir), is_relative_path=False) input_files = storage.list_dirs(fname_url_with_pkey, list_files=True) # get process information run_counter = context['run_counter'] logger.debug("run_counter=%s" % run_counter) proc = None for p in processes: # TODO: how to handle invalid run_counter pid = int(p['id']) logger.debug("pid=%s" % pid) if pid == run_counter: proc = p break else: logger.error("no process found matching run_counter") raise BadInputException() logger.debug("proc=%s" % pformat(proc)) for fname in input_files: logger.debug("fname=%s" % fname) templ_mat = template_pat.match(fname) fname_url_with_credentials = storage.get_url_with_credentials( output_storage_settings, output_prefix + os.path.join(self.iter_inputdir, input_dir, fname), is_relative_path=False) logger.debug("fname_url_with_credentials=%s" % fname_url_with_credentials) def put_dest_file(proc, fname, dest_file_location, resched_file_location, content): dest_url = get_url_with_credentials( computation_platform_settings, os.path.join(dest_file_location, fname), is_relative_path=True, ip_address=proc['ip_address']) logger.debug("writing to =%s" % dest_url) #logger.debug("content=%s" % content) storage.put_file(dest_url, content) if self.reschedule_failed_procs: logger.debug("resched=%s" % resched_file_location) logger.debug("fname=%s" % fname) logger.debug("output_storage_settings=%s" % output_storage_settings) logger.debug("here") test = "%s/%s" % (resched_file_location, fname) logger.debug("test=%s" % test) resched_url = get_url_with_credentials( output_storage_settings, test) logger.debug("writing backup to %s" % resched_url) storage.put_file(resched_url, content) logger.debug("done") outputs = [] if templ_mat: base_fname = templ_mat.group(1) template_content = storage.get_file( fname_url_with_credentials) try: templ = Template(template_content) except TemplateSyntaxError, e: logger.error(e) #FIXME: should detect this during submission of job, #as no sensible way to recover here. #TODO: signal error conditions in job status continue new_context = Context(context) logger.debug("new_content=%s" % new_context) render_output = templ.render(new_context) render_output = render_output.encode('utf-8') outputs.append((base_fname, render_output)) outputs.append((fname, template_content)) else: content = storage.get_file(fname_url_with_credentials) outputs.append((fname, content)) for (new_fname, content) in outputs: dest_file_location = computation_platform_settings['type']\ + "@" + os.path.join(relative_path_suffix, proc['id'], local_settings['payload_cloud_dirname']) logger.debug("dest_file_location =%s" % dest_file_location) resched_file_location = "%s%s" % ( output_prefix, os.path.join(self.job_dir, "input_backup", proc['id'])) logger.debug("resched_file_location=%s" % resched_file_location) put_dest_file(proc, new_fname, dest_file_location, resched_file_location, content) # then copy context new values file logger.debug("writing values file") values_dest_location = computation_platform_settings['type']\ + "@" + os.path.join(relative_path_suffix, proc['id'], local_settings['payload_cloud_dirname'], self.VALUES_FNAME) logger.debug("values_dest_location =%s" % values_dest_location) values_dest_url = get_url_with_credentials( computation_platform_settings, values_dest_location, is_relative_path=True, ip_address=proc['ip_address']) storage.put_file(values_dest_url, json.dumps(context, indent=4))
logger.debug("values_url=%s" % values_url) values_content = get_file(values_url) logger.debug("values_content=%s" % values_content) v_map = dict(json.loads(values_content), ) except IOError: logger.warn("no values file found") except ValueError: logger.error("problem parsing contents of %s" % VALUES_MAP_FILE) pass v_map.update(starting_map) v_map.update(context) v_map['run_counter'] = 1 logger.debug("new v_map=%s" % v_map) put_file(values_url, json.dumps(v_map, indent=4)) # Set random numbers for subdirective logger.debug("run_settings=%s" % pformat(run_settings)) if rands: setval(run_settings, '%s/input/hrmc/iseed' % django_settings.SCHEMA_PREFIX, rands[i]) if input_loc: # Set revised input_location for subdirective setval( run_settings, input_loc, "%s/%s/%s" % (self.scratch_platform, SUBDIRECTIVE_DIR % { 'run_counter': str(run_counter) }, FIRST_ITERATION_DIR))