'mem': '250g', 'walltime': '4000:00:00', 'ppn': 24, 'gpus': 2 } job_settings.update({ 'outfile':os.path.join(jobdir, '$PBS_JOBID.out'), 'errfile':os.path.join(jobdir, '$PBS_JOBID.err'), 'email': '*****@*****.**', 'email_options': 'a' }) arg_to_pass = '' PythonJob( python_file, python_executable = '/home/jeromel/.conda/envs/deep_work_gpu/bin/python', conda_env = 'deep_work_gpu', jobname = 'movie_2p', python_args = arg_to_pass+' > '+output_terminal, **job_settings ).run(dryrun=False)
def main(argv): opts, args = getopt.getopt( argv, [], [ "output_folder=", "model_file=", "dat_file=", ], ) for opt, arg in opts: if opt == "--output_folder": output_folder = arg if opt == "--model_file": model_file = arg if opt == "--dat_file": dat_file = arg try: os.mkdir(output_folder) except Exception: print("Folder already created") # We fist fine-tune the model python_file = (r"/home/jeromel/Documents/Projects/Deep2P/repos/" + r"deepinterpolation/examples/paper_generation_code/" + r"cluster_lib/single_ephys_transfer_trainer.py") now = datetime.datetime.now() run_uid = now.strftime("%Y_%m_%d_%H_%M") output_terminal = os.path.join( output_folder, run_uid+'_running_terminal.txt') nb_probes = 384 raw_data = np.memmap(dat_file, dtype="int16") img_per_movie = int(raw_data.size / nb_probes) pre_post_frame = 30 batch_size = 100 nb_jobs = 100 training_samples = 10000000 pre_post_omission = 1 start_frame = pre_post_omission + pre_post_frame end_frame = img_per_movie - pre_post_frame - pre_post_omission - 1 job_settings = {'queue': 'braintv', 'mem': '250g', 'walltime': '48:00:00', 'ppn': 24, 'gpus': 1 } job_settings.update({ 'outfile': os.path.join(output_folder, '$PBS_JOBID.out'), 'errfile': os.path.join(output_folder, '$PBS_JOBID.err'), 'email': '*****@*****.**', 'email_options': 'a' }) arg_to_pass = [ "--movie_path " + dat_file + " --train_frame_start " + str(20000) + " --train_frame_end " + str(-1) + " --train_total_samples " + str(training_samples) + " --val_frame_start " + str(0) + " --val_frame_end " + str(19999) + " --val_total_samples " + str(-1) + " --output_path " + output_folder + " --model_file " + model_file + " --batch_size " + str(batch_size) + " --pre_post_frame " + str(pre_post_frame) + " --pre_post_omission " + str(pre_post_omission) + " --loss " + 'mean_squared_error' ] PythonJob( python_file, python_executable=(r"/allen/programs/braintv/workgroups/nc-ophys/" + r"Jeromel/conda/tf20-env/bin/python"), conda_env=(r"/allen/programs/braintv/workgroups/nc-ophys/Jeromel/" + r"conda/tf20-env"), jobname='fine_tuning_ephys', python_args=arg_to_pass[0] + ' > ' + output_terminal, **job_settings ).run(dryrun=False) # We wait for the jobs to complete stay_in_loop = True while stay_in_loop: time.sleep(60) list_files = glob.glob(os.path.join(output_folder, "*_model.h5")) if len(list_files) > 0: stay_in_loop = False new_model_file = list_files[0] # We infer the movie in chunks block_size = np.int(np.ceil((end_frame - start_frame) / nb_jobs)) # We force block_size to be a multiple of batch size block_size = int(np.floor(block_size / batch_size) * batch_size) jobdir = os.path.join( output_folder, "tmp_" + os.path.splitext(os.path.basename(model_file))[0] ) try: os.mkdir(jobdir) except Exception: print("Folder already created") files = glob.glob(os.path.join(jobdir, "*")) for f in files: os.remove(f) python_file = (r"/home/jeromel/Documents/Projects/Deep2P/repos/" + r"deepinterpolation/examples/paper_generation_code/" + r"cluster_lib/single_ephys_section_inferrence.py") list_files_check = [] for index, local_start_frame in enumerate( np.arange(start_frame, end_frame, block_size) ): local_path = os.path.join(jobdir, "movie_" + str(index) + ".hdf5") local_end_frame = np.min([end_frame, local_start_frame + block_size - 1]) job_settings = { "queue": "braintv", "mem": "180g", "walltime": "48:00:00", "ppn": 16, } out_file = os.path.join(jobdir, "$PBS_JOBID.out") list_files_check.append(local_path + ".done") job_settings.update( { "outfile": out_file, "errfile": os.path.join(jobdir, "$PBS_JOBID.err"), "email": "*****@*****.**", "email_options": "a", } ) arg_to_pass = [ "--movie_path " + dat_file + " --frame_start " + str(local_start_frame) + " --frame_end " + str(local_end_frame) + " --output_file " + local_path + " --model_file " + new_model_file + " --batch_size " + str(batch_size) + " --pre_post_frame " + str(pre_post_frame) + " --pre_post_omission " + str(pre_post_omission) ] PythonJob( python_file, python_executable=(r"/allen/programs/braintv/workgroups/" + r"nc-ophys/Jeromel/conda/tf20-env/bin/python"), conda_env=(r"/allen/programs/braintv/workgroups/nc-ophys/" + r"Jeromel/conda/tf20-env"), jobname="ephys_inferrence", python_args=arg_to_pass[0], **job_settings ).run(dryrun=False) # We wait for the jobs to complete stay_in_loop = True while stay_in_loop: time.sleep(60) nb_file = 0 for indiv_file in list_files_check: if os.path.isfile(indiv_file): nb_file += 1 if nb_file == len(list_files_check): stay_in_loop = False # We merge the files output_merged = os.path.join(output_folder, "movie_" + os.path.basename(model_file)) list_files = glob.glob(os.path.join(jobdir, "*.hdf5")) list_files = sorted( list_files, key=lambda x: int(x.split("movie_")[1].split(".hdf5")[0]) ) nb_frames = 0 for each_file in list_files: with h5py.File(each_file, "r") as file_handle: local_shape = file_handle["data"].shape nb_frames = nb_frames + local_shape[0] final_shape = list(local_shape) final_shape[0] = nb_frames global_index_frame = 0 with h5py.File(output_merged, "w") as file_handle: dset_out = file_handle.create_dataset( "data", shape=final_shape, chunks=(1, final_shape[1], final_shape[2], final_shape[3]), dtype="float16", ) for each_file in list_files: with h5py.File(each_file, "r") as file_handle: local_shape = file_handle["data"].shape dset_out[ global_index_frame: global_index_frame + local_shape[0], :, :, : ] = file_handle["data"][:, :, :, :] global_index_frame += local_shape[0] shutil.rmtree(jobdir)
job_settings = { 'queue': 'braintv', 'mem': '15g', 'walltime': '0:30:00', 'ppn': 1, 'jobdir': jobdir, } cache_json = { 'manifest_path': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv', 'nwb_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files', 'analysis_files_base_dir': '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files' } cache = bpc.BehaviorProjectCache(cache_json) experiment_ids = cache.manifest['ophys_experiment_id'].values for experiment_id in experiment_ids: PythonJob( python_file, python_executable='/home/nick.ponvert/anaconda3/envs/allen/bin/python', python_args=experiment_id, conda_env=None, jobname='trial_response_df_{}'.format(experiment_id), **job_settings).run(dryrun=False)
run_params["case"] = case if net_name is not None: run_params["net_name"] = net_name # prepare args params_list = list( chain.from_iterable( (f"--{k}", str(run_params[k])) for k in run_params)) params_string = " ".join(params_list) # kick off HPC job PythonJob(script, python_executable, conda_env=conda_env, python_args=params_string, jobname=job_title, jobdir=job_dir, **job_settings).run(dryrun=False) def param_arr_helper(param_arr): if param_arr is None or len(param_arr) == 0: return None return " ".join(str(p) for p in param_arr) if __name__ == "__main__": args = parser.parse_args()
# 913106887, 913106889, 913106893, 913106896] #VisualBehaviorTask1B 7/29/19 experiment_ids = [ 899085531, 901559828, 901560861, 902474078, 902487182, 902487200, 902490609, 903396212, 903403819, 903405627, 904155140, 904155155, 904161089, 905495248, 908350502, 908350518, 909090479, 910213154, 910600099, 910608808, 911146705, 911149119, 911149136, 911642167, 911642184 ] python_file = r"/home/marinag/visual_behavior_analysis/scripts/convert_level_1_to_level_2.py" jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords2' job_settings = { 'queue': 'braintv', 'mem': '30g', 'walltime': '5:00:00', 'ppn': 1, 'jobdir': jobdir, } for experiment_id in experiment_ids: PythonJob(python_file, python_executable= '/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python', python_args=experiment_id, conda_env=None, jobname='process_{}'.format(experiment_id), **job_settings).run(dryrun=False)
jobdir, run_uid + os.path.basename(python_file) + "_running_terminal.txt" ) job_settings = { "queue": "braintv", "mem": "250g", "walltime": "200:00:00", "ppn": 16, "gpus": 1, } job_settings.update( { "outfile": os.path.join(jobdir, "$PBS_JOBID.out"), "errfile": os.path.join(jobdir, "$PBS_JOBID.err"), "email": "*****@*****.**", "email_options": "a", } ) arg_to_pass = "" PythonJob( python_file, python_executable="/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env/bin/python", # "/home/jeromel/.conda/envs/deep_work_gpu/bin/python", conda_env="/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env", # "deep_work_gpu", jobname="tf_" + os.path.basename(python_file), python_args=arg_to_pass + " > " + output_terminal, **job_settings ).run(dryrun=False)
def main(argv): opts, args = getopt.getopt( argv, [], [ "output_folder=", "model_file=", "start_frame=", "end_frame=", "pre_post_frame=", "nb_jobs=", "dat_file=", "pre_post_omission=", ], ) for opt, arg in opts: if opt == "--output_folder": output_folder = arg if opt == "--model_file": model_file = arg if opt == "--start_frame": start_frame = int(arg) if opt == "--end_frame": end_frame = int(arg) if opt == "--pre_post_frame": pre_post_frame = int(arg) if opt == "--nb_jobs": nb_jobs = int(arg) if opt == "--dat_file": dat_file = arg if opt == "--pre_post_omission": pre_post_omission = int(arg) try: os.mkdir(output_folder) except: print("Folder already created") batch_size = 200 # We infer the movie in chunks block_size = np.int(np.ceil((end_frame - start_frame) / nb_jobs)) # We force block_size to be a multiple of batch size block_size = int(np.floor(block_size / batch_size) * batch_size) jobdir = os.path.join( output_folder, "tmp_" + os.path.splitext(os.path.basename(model_file))[0] ) try: os.mkdir(jobdir) except: print("Folder already created") files = glob.glob(os.path.join(jobdir, "*")) for f in files: os.remove(f) python_file = "/home/jeromel/Documents/Projects/Deep2P/repos/deepinterpolation/examples/cluster_lib/single_ephys_section_inferrence.py" list_files_check = [] for index, local_start_frame in enumerate( np.arange(start_frame, end_frame, block_size) ): local_path = os.path.join(jobdir, "movie_" + str(index) + ".hdf5") local_end_frame = np.min([end_frame, local_start_frame + block_size - 1]) job_settings = { "queue": "braintv", "mem": "180g", "walltime": "48:00:00", "ppn": 16, } out_file = os.path.join(jobdir, "$PBS_JOBID.out") list_files_check.append(local_path + ".done") job_settings.update( { "outfile": out_file, "errfile": os.path.join(jobdir, "$PBS_JOBID.err"), "email": "*****@*****.**", "email_options": "a", } ) arg_to_pass = [ "--movie_path " + dat_file + " --frame_start " + str(local_start_frame) + " --frame_end " + str(local_end_frame) + " --output_file " + local_path + " --model_file " + model_file + " --batch_size " + str(batch_size) + " --pre_post_frame " + str(pre_post_frame) + " --pre_post_omission " + str(pre_post_omission) ] PythonJob( python_file, python_executable="/home/jeromel/.conda/envs/deep_work2/bin/python", conda_env="deep_work2", jobname="movie_2p", python_args=arg_to_pass[0], **job_settings ).run(dryrun=False) # We wait for the jobs to complete stay_in_loop = True while stay_in_loop: time.sleep(60) nb_file = 0 for indiv_file in list_files_check: if os.path.isfile(indiv_file): nb_file += 1 if nb_file == len(list_files_check): stay_in_loop = False # We merge the files output_merged = os.path.join(output_folder, "movie_" + os.path.basename(model_file)) list_files = glob.glob(os.path.join(jobdir, "*.hdf5")) list_files = sorted( list_files, key=lambda x: int(x.split("movie_")[1].split(".hdf5")[0]) ) nb_frames = 0 for each_file in list_files: with h5py.File(each_file, "r") as file_handle: local_shape = file_handle["data"].shape nb_frames = nb_frames + local_shape[0] final_shape = list(local_shape) final_shape[0] = nb_frames global_index_frame = 0 with h5py.File(output_merged, "w") as file_handle: dset_out = file_handle.create_dataset( "data", shape=final_shape, chunks=(1, final_shape[1], final_shape[2], final_shape[3]), dtype="float16", ) for each_file in list_files: with h5py.File(each_file, "r") as file_handle: local_shape = file_handle["data"].shape dset_out[ global_index_frame : global_index_frame + local_shape[0], :, :, : ] = file_handle["data"][:, :, :, :] global_index_frame += local_shape[0] shutil.rmtree(jobdir)
'email_options': 'a' }) jobname = 'sg_mod_aapc' units_savename = os.path.join(units_savedir,expt_id + '_units_df.pkl') units_df = pd.read_pickle(units_savename) jobcount = 0 for cstr in cstr_list: visp_units = units_df[units_df.ecephys_structure_acronym ==cstr].index.values no_of_jobs = len(visp_units) * len(stim_scale_list) while (jobcount + no_of_jobs) >= 8000: time.sleep(30) out = subprocess.Popen(["qstat", "-u rami"], stdout=subprocess.PIPE) out2 = subprocess.Popen(["wc", "-l"], stdin=out.stdout, stdout=subprocess.PIPE).communicate()[0].strip('\n') jobcount = int(float(out2)) if len(visp_units) > 0: jobcount = jobcount + no_of_jobs for unit_id in visp_units: for stim_scale in stim_scale_list: job_settings['jobname'] = '%s:%s:%s:%s:%s' % (jobname, expt_id, cstr, unit_id, stim_scale) PythonJob(os.path.join(basedir,'sandbox_pop_coupling_with_modules.py'), python_args='--cstr=%s --expt_id=%s --unit_id=%s --prs_yaml_fname=%s --stim_scale=%s' % (cstr, expt_id, unit_id, prs_yaml_fname, stim_scale), conda_env=conda_env, python_path=None, **job_settings).run(dryrun=dryrun)
def main(net_name, config_groups, scheme, resume, lr, lr_step_size, lr_gamma, batch_size, dataset): job_title = "train_net" # script, run_params and job_settings with open("job_params.json", "r") as json_file: job_params = json.load(json_file) with open("net_configs.json", "r") as json_file: net_configs = json.load(json_file) # get cases in current groups cases = set() for group in config_groups: configs = net_configs[group] for case in configs.keys(): cases.add(case) job_params = job_params[job_title] script = job_params["script"] # update any run params run_params = job_params["run_params"] run_params["net_name"] = net_name run_params["scheme"] = scheme run_params["dataset"] = dataset if lr is not None: run_params["lr"] = lr run_params[ "lr_step_size"] = lr_step_size if lr_step_size is not None else run_params[ "lr_step_size"] run_params["lr_gamma"] = lr_gamma if lr_gamma is not None else run_params[ "lr_gamma"] run_params[ "batch_size"] = batch_size if batch_size is not None else run_params[ "batch_size"] job_settings = job_params["job_settings"] # set to avoid submitting jobs for the same net twice net_filepaths = set() # walk dir looking for nets to train net_dir = os.path.join(run_params["data_dir"], f"nets/{dataset}/{net_name}") for root, dirs, files in os.walk(net_dir): # only interested in locations files (nets) are saved if len(files) <= 0: continue slugs = root.split("/") # only interested in the given dataset if not dataset in slugs: continue # only interested in the given training scheme if not scheme in slugs: continue # only interested in the given groups... if not any(g in slugs for g in config_groups): continue # ...and their cases if not any(c in slugs for c in cases): continue # start from first or last epoch if resume: net_filename = get_last_epoch(files) print(f"Submitting job to resume training of {net_filename}.") else: net_filename = get_first_epoch(files) print(f"Job will begin from initial snapshot {net_filename}.") # and add it to the training job set net_filepath = os.path.join(root, net_filename) net_filepaths.add(net_filepath) # loop over set, submitting jobs for net_filepath in net_filepaths: # update param run_params["net_filepath"] = net_filepath # prepare args params_list = list( chain.from_iterable( (f"--{k}", str(run_params[k])) for k in run_params)) params_string = " ".join(params_list) # kick off HPC job PythonJob(script, python_executable, conda_env=conda_env, python_args=params_string, jobname=job_title + f" {net_filepath}", jobdir=job_dir, **job_settings).run(dryrun=False)
from pbstools import PythonJob # flake8: noqa: E999 import visual_behavior.data_access.loading as loading python_file = r"/home/marinag/visual_behavior_analysis/scripts/concatenate_stimulus_presentations.py" jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords' job_settings = { 'queue': 'braintv', 'mem': '50g', 'walltime': '10:00:00', 'ppn': 1, 'jobdir': jobdir, } experiments_table = loading.get_filtered_ophys_experiment_table() for project_code in experiments_table.project_code.unique(): for session_number in experiments_table.session_number.unique(): PythonJob( python_file, python_executable= '/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python', python_args=[project_code, session_number], conda_env=None, jobname='stimulus_presentations_' + project_code + '_' + str(session_number), **job_settings).run(dryrun=False)
import sys import platform if platform.system() == 'Linux': # sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/Doug/pbstools') sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/nick.ponvert/src/pbstools') from pbstools import PythonJob # flake8: noqa: E999 python_file = r"/home/marinag/visual_behavior_analysis/scripts/create_multi_session_mean_df.py" jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords2' job_settings = {'queue': 'braintv', 'mem': '200g', 'walltime': '40:00:00', 'ppn': 1, 'jobdir': jobdir, } PythonJob( python_file, python_executable='/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python', python_args=None, conda_env=None, jobname='process_multi_session_df', **job_settings ).run(dryrun=False)
"mem": "180g", "walltime": "12:00:00", "ppn": 16, } out_file = os.path.join(jobdir, "$PBS_JOBID.out") job_settings.update({ "outfile": out_file, "errfile": os.path.join(jobdir, "$PBS_JOBID.err"), "email": "*****@*****.**", "email_options": "a", }) arg_to_pass = [ "--json_path " + json_path + " --output_file " + local_path + " --model_file " + os.path.join(access_to_models, model_file) + " --batch_size " + str(batch_size) + " --pre_frame " + str(pre_frame) + " --post_frame " + str(post_frame) ] PythonJob( python_file, python_executable= "/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env/bin/python", conda_env= "/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env", jobname="movie_2p", python_args=arg_to_pass[0], **job_settings).run(dryrun=False)
job_settings = { "queue": "braintv", "mem": "250g", "walltime": "24:00:00", "ppn": 16, } job_settings.update({ "outfile": os.path.join(jobdir, "$PBS_JOBID.out"), "errfile": os.path.join(jobdir, "$PBS_JOBID.err"), "email": "*****@*****.**", "email_options": "a", }) arg_to_pass = (" --dat_file " + dat_file + " --output_folder " + output_folder + " --model_file " + model_file) arg_to_pass += (" --start_frame " + str(start_frame) + " --end_frame " + str(end_frame) + " --pre_post_frame " + str(pre_post_frame)) arg_to_pass += (" --nb_jobs " + str(nb_jobs) + " --pre_post_omission " + str(pre_post_omission)) PythonJob( python_file, python_executable="/home/jeromel/.conda/envs/deep_work2/bin/python", conda_env="deep_work2", jobname="movie_2p", python_args=arg_to_pass + " > " + output_terminal, **job_settings).run(dryrun=False)
def main(dataset, net_names, schemes, config_groups, find_lr_avg): job_title = "gen_nets" # script, run_params and job_settings with open("job_params.json", "r") as json_file: job_params = json.load(json_file) with open("net_configs.json", "r") as json_file: net_configs = json.load(json_file) job_params = job_params[job_title] script = job_params["script"] run_params = job_params["run_params"] run_params["dataset"] = dataset job_settings = job_params["job_settings"] # kick off job for each net configuration for net_name in net_names: for scheme in schemes: for group in config_groups: # get net configs in group configs = net_configs[group] for case in configs.keys(): # get net config details config = configs[case] # update params for this net config run_params["net_name"] = net_name run_params["scheme"] = scheme run_params["group"] = group run_params["case"] = case if config.get("conv_layers") is not None: run_params["conv_layers"] = config["conv_layers"] if config.get("fc_layers") is not None: run_params["fc_layers"] = config["fc_layers"] if config.get("default_fn") is not None: run_params["default_fn"] = config["default_fn"] act_fns = config["act_fns"] run_params["act_fns"] = param_arr_helper(act_fns) if config.get("act_fn_params") is not None: run_params["act_fn_params"] = param_arr_helper(config.get("act_fn_params")) # default repeating each fn once if config.get("n_repeat") is None: n_repeat = [1] * len(act_fns) else: n_repeat = config.get("n_repeat") run_params["n_repeat"] = param_arr_helper(n_repeat) # prepare args params_list = list(chain.from_iterable((f"--{k}", str(run_params[k])) for k in run_params)) pretrained = config.get("pretrained") if pretrained: params_list.append("--pretrained") spatial = config.get("spatial") if spatial: params_list.append("--spatial") cfg_find_lr_avg = config.get("find_lr_avg") if cfg_find_lr_avg or find_lr_avg: params_list.append("--find_lr_avg") params_string = " ".join(params_list) # kick off HPC job PythonJob( script, python_executable, conda_env = conda_env, python_args = params_string, jobname = job_title + f" c-{case}", jobdir = job_dir, **job_settings ).run(dryrun=False)
def main(net_name, cases, scheme, dataset, final): job_title = "save_net_activations" # script, run_params and job_settings with open("job_params.json", "r") as json_file: job_params = json.load(json_file) with open("net_configs.json", "r") as json_file: net_configs = json.load(json_file) job_params = job_params[job_title] script = job_params["script"] # update any run params run_params = job_params["run_params"] run_params["net_name"] = net_name run_params["dataset"] = dataset job_settings = job_params["job_settings"] # set to avoid submitting jobs for the same net twice net_filepaths = set() # walk dir looking for nets to train net_dir = os.path.join(run_params["data_dir"], f"nets/{dataset}/{net_name}/{scheme}") for root, dirs, files in os.walk(net_dir): # only interested in locations files (nets) are saved if len(files) <= 0: continue slugs = root.split("/") # only interested in the given dataset if not dataset in slugs: continue # only interested in the given training scheme if not scheme in slugs: continue # only interested in the given cases (unless []) if len(cases) > 0 and not any(c in slugs for c in cases): continue # start from first or last epoch if final: net_filename = get_last_epoch(files) print( f"Submitting job to save activation output of {net_filename}.") else: net_filename = get_first_epoch(files) print( f"Job will save activations of initial snapshot {net_filename}." ) # and add it to the training job set net_filepath = os.path.join(root, net_filename) net_filepaths.add(net_filepath) # loop over set, submitting jobs for net_filepath in net_filepaths: # update param run_params["net_filepath"] = net_filepath # prepare args params_list = list( chain.from_iterable( (f"--{k}", str(run_params[k])) for k in run_params)) params_string = " ".join(params_list) # kick off HPC job PythonJob(script, python_executable, conda_env=conda_env, python_args=params_string, jobname=job_title + f" {net_filepath}", jobdir=job_dir, **job_settings).run(dryrun=False)