def run_batch(qlogger, msg_type, roles, msg_data, work_dir, storage_creds, store_in_cloud, log_str):
qlogger.srvr_log(logging.INFO, "beginning run_batch")
cwd = os.getcwd()
store = GoogleDriveStore(storage_creds, log_str) if (None != storage_creds) else None
if store_in_cloud:
batch_zip = msg_data
batch_zip_name = os.path.splitext(store.to_file_name(batch_zip))[0] + '_output.zip'
qlogger.clnt_log("Reading from cloud store: " + batch_zip)
batch_folder = Utils.mkdtemp(prefix="batch_")
batch_zip = store.copy_to_local(batch_zip, work_dir)
Utils.uncompress_folder(batch_folder, batch_zip)
os.remove(batch_zip)
output_folder_zip = os.path.join(work_dir, batch_zip_name)
else:
batch_folder = msg_data
qlogger.clnt_log("Running batch with all configurations (.ini files) under folder: " + batch_folder)
output_root_folder = Utils.mkdtemp_if_exists(prefix="output", dir=batch_folder)
if store_in_cloud:
qlogger.clnt_log("Outputs would be uploaded to your cloud store as " + batch_zip_name)
else:
qlogger.clnt_log("Outputs would be stored under folder: " + output_root_folder)
num_success = 0
num_failed = 0
# collect all configuration names
config_files = []
for root, _dirs, files in os.walk(batch_folder):
for file_name in files:
if not file_name.endswith(".ini"):
continue
config_files.append(os.path.join(root, file_name))
num_configs = len(config_files)
qlogger.clnt_log("Found " + str(num_configs) + " configuration files.")
parallelize = True if (num_configs > CircuitscapeRunner.MAX_PARALLEL) else False
batch_success = False
if parallelize:
pool = []
results = []
try:
for config_file in config_files:
root, file_name = os.path.split(config_file)
qlogger.clnt_log("Loading configuration: " + config_file)
cfg = CSConfig(config_file)
os.chdir(root)
qlogger.clnt_log("Verifying configuration...")
if parallelize: # switch off parallization of each task if we are parallelizing batch
cfg.parallelize = False
(all_options_valid, message) = cfg.check()
#qlogger.clnt_log("Verified configuration with result: " + str(all_options_valid))
if all_options_valid:
#qlogger.clnt_log("Verifying all paths are relative...")
all_options_valid = cfg.are_all_paths_relative();
if not all_options_valid:
message = "All file paths in configuration must be relative to location of configuration file."
#qlogger.clnt_log("Verified configuration with result: " + str(all_options_valid))
if all_options_valid:
qlogger.clnt_log("Verifying profile limits...")
(all_options_valid, message) = CircuitscapeRunner.check_role_limits(roles, cfg, qlogger)
if not all_options_valid:
qlogger.send_error_msg(message)
num_failed += 1
else:
solver_failed = True
cfgname = os.path.splitext(file_name)[0]
output_folder = os.path.join(output_root_folder, cfgname)
os.mkdir(output_folder)
cfg.output_file = os.path.join(output_folder, os.path.basename(cfg.output_file))
outdir, _out_file = os.path.split(cfg.output_file)
try:
qlogger.clnt_log("Storing final configuration...")
configFile = os.path.join(outdir, 'circuitscape.ini')
cfg.write(configFile)
if parallelize:
if len(pool) >= CircuitscapeRunner.MAX_PARALLEL:
p = pool.pop()
p.join()
result = results.pop()
if result.value == 0:
num_success += 1
else:
num_failed += 1
result = Value('i', -1)
p = Process(target=CircuitscapeRunner._run_compute, args=(configFile, qlogger, cfgname + ' => ', result))
pool.append(p)
results.append(result)
p.start()
else:
cs = Compute(configFile, qlogger)
result, solver_failed = cs.compute()
qlogger.clnt_log("Result: \n" + str(result))
except Exception as e:
message = str(e)
qlogger.send_error_msg(message)
if not parallelize:
if solver_failed:
num_failed += 1
else:
num_success += 1
if parallelize:
qlogger.srvr_log(logging.DEBUG, "waiting for processes len=" + str(len(pool)))
for idx in range(0, len(pool)):
pool[idx].join()
if results[idx].value == 0:
num_success += 1
else:
num_failed += 1
qlogger.srvr_log(logging.DEBUG, "Batch run done for " + str(len(config_files)) + " configuration files. Success: " + str(num_success) + ". Failures: " + str(num_failed))
qlogger.clnt_log("Batch run done for " + str(len(config_files)) + " configuration files. Success: " + str(num_success) + ". Failures: " + str(num_failed))
if num_success > 0:
if store_in_cloud:
CircuitscapeRunner.upload_results(qlogger, output_root_folder, output_folder_zip, work_dir, msg_data, store, extract_folder_id=True)
else:
qlogger.clnt_log("Outputs under folder: " + output_root_folder)
batch_success = True
except Exception as e:
qlogger.clnt_log("Unexpected error during batch run.")
qlogger.srvr_log(logging.WARNING, "Exception during batch run: " + str(e))
finally:
os.chdir(cwd)
if store_in_cloud:
try:
Utils.rmdir(batch_folder)
if os.path.exists(output_folder_zip):
os.remove(output_folder_zip)
except:
qlogger.srvr_log(logging.ERROR, "Could not clear temporary files.")
qlogger.srvr_log(logging.INFO, "end run_batch")
qlogger.send_result_msg(msg_type, {'complete': True, 'success': batch_success})
def run_job(qlogger, msg_type, roles, msg_data, work_dir, storage_creds, store_in_cloud, log_str):
qlogger.srvr_log(logging.INFO, "beginning run_job")
solver_failed = True
output_cloud_folder = None
output_folder = None
cfg = CSConfig()
store = GoogleDriveStore(storage_creds, log_str) if (None != storage_creds) else None
all_options_valid = True
for key in msg_data.keys():
val = msg_data[key]
if store_in_cloud and (key in CSConfig.FILE_PATH_PROPS) and (val != None):
# if val is gdrive location, translate it to local drive
if isinstance(val, str) or isinstance(val, unicode):
val = val.strip()
if val.startswith("gdrive://"):
if key == 'output_file':
# store the output gdrive folder
qlogger.clnt_log("Preparing cloud store output folder: " + val)
output_cloud_folder = val
output_folder = os.path.join(work_dir, 'output')
if not os.path.exists(output_folder):
os.mkdir(output_folder)
else:
Utils.rmdir(output_folder, True)
val = os.path.join(output_folder, 'results.out')
else:
# copy the file locally
qlogger.clnt_log("Reading from cloud store: " + val)
val = store.copy_to_local(val, work_dir)
elif (len(val) > 0):
message = "Configuration parameter %s must have a gdrive URL as value. Current value: '%s'"%(key, val)
all_options_valid = False
break
elif key == 'output_file':
message = "Output location not specified."
all_options_valid = False
break
cfg.__setattr__(key, val)
if all_options_valid:
qlogger.clnt_log("Verifying configuration...")
(all_options_valid, message) = cfg.check()
if all_options_valid:
qlogger.clnt_log("Verifying profile limits...")
(all_options_valid, message) = CircuitscapeRunner.check_role_limits(roles, cfg, qlogger)
if not all_options_valid:
qlogger.send_error_msg(message)
else:
# In cloud mode, this would be a temporary directory
outdir, _out_file = os.path.split(cfg.output_file)
try:
qlogger.clnt_log("Storing final configuration...")
configFile = os.path.join(outdir, 'circuitscape.ini')
cfg.write(configFile)
cs = Compute(configFile, qlogger)
result, solver_failed = cs.compute()
qlogger.clnt_log("Result: \n" + str(result))
except Exception as e:
message = str(e)
qlogger.send_error_msg(message)
success = not solver_failed
if success and store_in_cloud:
output_folder_zip = os.path.join(work_dir, 'output.zip')
CircuitscapeRunner.upload_results(qlogger, output_folder, output_folder_zip, work_dir, output_cloud_folder, store, extract_folder_id=False)
qlogger.srvr_log(logging.INFO, "end run_job")
qlogger.send_result_msg(msg_type, {'complete': True, 'success': success})