def process_inputs(self): """ Initiates Processing On Previously Acquired EC2 Instance. This version requires that you include a config (fourth) argument """ try: os.environ['COMMAND'].format("a","b","c","d") except IndexError as ie: msg = " [Internal (process_inputs)] INPUT ERROR: not enough arguments in the COMMAND argument." self.logger.append(msg) self.logger.printlatest() self.logger.write() raise ValueError("[JOB TERMINATE REASON] Not the correct format for arguments. Protocols for job manager are misformatted.") ## Should we vectorize the log here? outpath_full = os.path.join(os.environ['OUTDIR'],self.jobname) print([os.environ['COMMAND'].format( self.bucket_name, filename, outpath_full, self.config_name ) for filename in self.filenames],"command send") for f,filename in enumerate(self.filenames): response = utilsparamssm.execute_commands_on_linux_instances( commands=[os.environ['COMMAND'].format( self.bucket_name, filename, outpath_full, self.config_name )], # TODO: variable outdir as option instance_ids=[self.instances[f].instance_id], working_dirs=[os.environ['WORKING_DIRECTORY']], log_bucket_name=self.bucket_name, log_path=os.path.join(self.jobpath,'internal_ec2_logs') ) self.logger.initialize_datasets_dev(filename,self.instances[f].instance_id,response["Command"]["CommandId"]) self.logger.append(" [Internal (process_inputs)] Starting analysis {} with parameter set {}".format(f+1,os.path.basename(filename))) self.logger.printlatest() self.logger.write() self.logger.append(" [Internal (process_inputs)] All jobs submitted. Processing...")
def process_inputs(self): """ Initiates Processing On Previously Acquired EC2 Instance. This version requires that you include a config (fourth) argument """ print(self.input_bucket_name, 'bucket name') print(self.filenames, 'filenames') print(os.environ['OUTDIR'], 'outdir') print(os.environ['COMMAND'], 'command') try: os.environ['COMMAND'].format("a", "b", "c", "d") except IndexError as ie: msg = "not enough arguments in the COMMAND argument." self.submitlogger.append(msg) self.submitlogger.write() self.inputlogger.append(msg) self.inputlogger.write() raise ValueError("Not the correct format for arguments.") ## Should we vectorize the log here? outpath_full = os.path.join(os.environ['OUTDIR'], self.jobname) [ self.submitlogger.append("Sending command: {}".format( os.environ['COMMAND'].format(self.input_bucket_name, filename, outpath_full, self.config_name))) for filename in self.filenames ] [ self.inputlogger.append("Sending command: {}".format( os.environ['COMMAND'].format(self.input_bucket_name, filename, outpath_full, self.config_name))) for filename in self.filenames ] print([ os.environ['COMMAND'].format(self.input_bucket_name, filename, outpath_full, self.config_name) for filename in self.filenames ], "command sent") for f, filename in enumerate(self.filenames): response = utilsparamssm.execute_commands_on_linux_instances( commands=[ os.environ['COMMAND'].format(self.input_bucket_name, filename, outpath_full, self.config_name) ], # TODO: variable outdir as option instance_ids=[self.instances[f].instance_id], working_dirs=[os.environ['WORKING_DIRECTORY']], log_bucket_name=self.bucket_name, log_path=os.path.join(self.jobpath_submit, 'internal_ec2_logs')) #self.submitlogger.initialize_datasets_dev(filename,self.instances[f].instance_id,response["Command"]["CommandId"]) self.inputlogger.initialize_datasets_dev( filename, self.instances[f].instance_id, response["Command"]["CommandId"])