def process_inputs(self):
""" Initiates Processing On Previously Acquired EC2 Instance. This version requires that you include a config (fourth) argument """
try:
os.environ['COMMAND'].format("a","b","c","d")
except IndexError as ie:
msg = " [Internal (process_inputs)] INPUT ERROR: not enough arguments in the COMMAND argument."
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
raise ValueError("[JOB TERMINATE REASON] Not the correct format for arguments. Protocols for job manager are misformatted.")
## Should we vectorize the log here?
outpath_full = os.path.join(os.environ['OUTDIR'],self.jobname)
print([os.environ['COMMAND'].format(
self.bucket_name, filename, outpath_full, self.config_name
) for filename in self.filenames],"command send")
for f,filename in enumerate(self.filenames):
response = utilsparamssm.execute_commands_on_linux_instances(
commands=[os.environ['COMMAND'].format(
self.bucket_name, filename, outpath_full, self.config_name
)], # TODO: variable outdir as option
instance_ids=[self.instances[f].instance_id],
working_dirs=[os.environ['WORKING_DIRECTORY']],
log_bucket_name=self.bucket_name,
log_path=os.path.join(self.jobpath,'internal_ec2_logs')
)
self.logger.initialize_datasets_dev(filename,self.instances[f].instance_id,response["Command"]["CommandId"])
self.logger.append(" [Internal (process_inputs)] Starting analysis {} with parameter set {}".format(f+1,os.path.basename(filename)))
self.logger.printlatest()
self.logger.write()
self.logger.append(" [Internal (process_inputs)] All jobs submitted. Processing...")
def process_inputs(self):
""" Initiates Processing On Previously Acquired EC2 Instance. This version requires that you include a config (fourth) argument """
print(self.input_bucket_name, 'bucket name')
print(self.filenames, 'filenames')
print(os.environ['OUTDIR'], 'outdir')
print(os.environ['COMMAND'], 'command')
try:
os.environ['COMMAND'].format("a", "b", "c", "d")
except IndexError as ie:
msg = "not enough arguments in the COMMAND argument."
self.submitlogger.append(msg)
self.submitlogger.write()
self.inputlogger.append(msg)
self.inputlogger.write()
raise ValueError("Not the correct format for arguments.")
## Should we vectorize the log here?
outpath_full = os.path.join(os.environ['OUTDIR'], self.jobname)
[
self.submitlogger.append("Sending command: {}".format(
os.environ['COMMAND'].format(self.input_bucket_name, filename,
outpath_full, self.config_name)))
for filename in self.filenames
]
[
self.inputlogger.append("Sending command: {}".format(
os.environ['COMMAND'].format(self.input_bucket_name, filename,
outpath_full, self.config_name)))
for filename in self.filenames
]
print([
os.environ['COMMAND'].format(self.input_bucket_name, filename,
outpath_full, self.config_name)
for filename in self.filenames
], "command sent")
for f, filename in enumerate(self.filenames):
response = utilsparamssm.execute_commands_on_linux_instances(
commands=[
os.environ['COMMAND'].format(self.input_bucket_name,
filename, outpath_full,
self.config_name)
], # TODO: variable outdir as option
instance_ids=[self.instances[f].instance_id],
working_dirs=[os.environ['WORKING_DIRECTORY']],
log_bucket_name=self.bucket_name,
log_path=os.path.join(self.jobpath_submit,
'internal_ec2_logs'))
#self.submitlogger.initialize_datasets_dev(filename,self.instances[f].instance_id,response["Command"]["CommandId"])
self.inputlogger.initialize_datasets_dev(
filename, self.instances[f].instance_id,
response["Command"]["CommandId"])