'mem': '250g',
'walltime': '4000:00:00',
'ppn': 24,
'gpus': 2
}
job_settings.update({
'outfile':os.path.join(jobdir, '$PBS_JOBID.out'),
'errfile':os.path.join(jobdir, '$PBS_JOBID.err'),
'email': '*****@*****.**',
'email_options': 'a'
})
arg_to_pass = ''
PythonJob(
python_file,
python_executable = '/home/jeromel/.conda/envs/deep_work_gpu/bin/python',
conda_env = 'deep_work_gpu',
jobname = 'movie_2p',
python_args = arg_to_pass+' > '+output_terminal,
**job_settings
).run(dryrun=False)
def main(argv):
opts, args = getopt.getopt(
argv,
[],
[
"output_folder=",
"model_file=",
"dat_file=",
],
)
for opt, arg in opts:
if opt == "--output_folder":
output_folder = arg
if opt == "--model_file":
model_file = arg
if opt == "--dat_file":
dat_file = arg
try:
os.mkdir(output_folder)
except Exception:
print("Folder already created")
# We fist fine-tune the model
python_file = (r"/home/jeromel/Documents/Projects/Deep2P/repos/" +
r"deepinterpolation/examples/paper_generation_code/" +
r"cluster_lib/single_ephys_transfer_trainer.py")
now = datetime.datetime.now()
run_uid = now.strftime("%Y_%m_%d_%H_%M")
output_terminal = os.path.join(
output_folder, run_uid+'_running_terminal.txt')
nb_probes = 384
raw_data = np.memmap(dat_file, dtype="int16")
img_per_movie = int(raw_data.size / nb_probes)
pre_post_frame = 30
batch_size = 100
nb_jobs = 100
training_samples = 10000000
pre_post_omission = 1
start_frame = pre_post_omission + pre_post_frame
end_frame = img_per_movie - pre_post_frame - pre_post_omission - 1
job_settings = {'queue': 'braintv',
'mem': '250g',
'walltime': '48:00:00',
'ppn': 24,
'gpus': 1
}
job_settings.update({
'outfile': os.path.join(output_folder, '$PBS_JOBID.out'),
'errfile': os.path.join(output_folder, '$PBS_JOBID.err'),
'email': '*****@*****.**',
'email_options': 'a'
})
arg_to_pass = [
"--movie_path "
+ dat_file
+ " --train_frame_start "
+ str(20000)
+ " --train_frame_end "
+ str(-1)
+ " --train_total_samples "
+ str(training_samples)
+ " --val_frame_start "
+ str(0)
+ " --val_frame_end "
+ str(19999)
+ " --val_total_samples "
+ str(-1)
+ " --output_path "
+ output_folder
+ " --model_file "
+ model_file
+ " --batch_size "
+ str(batch_size)
+ " --pre_post_frame "
+ str(pre_post_frame)
+ " --pre_post_omission "
+ str(pre_post_omission)
+ " --loss "
+ 'mean_squared_error'
]
PythonJob(
python_file,
python_executable=(r"/allen/programs/braintv/workgroups/nc-ophys/" +
r"Jeromel/conda/tf20-env/bin/python"),
conda_env=(r"/allen/programs/braintv/workgroups/nc-ophys/Jeromel/" +
r"conda/tf20-env"),
jobname='fine_tuning_ephys',
python_args=arg_to_pass[0] + ' > ' + output_terminal,
**job_settings
).run(dryrun=False)
# We wait for the jobs to complete
stay_in_loop = True
while stay_in_loop:
time.sleep(60)
list_files = glob.glob(os.path.join(output_folder, "*_model.h5"))
if len(list_files) > 0:
stay_in_loop = False
new_model_file = list_files[0]
# We infer the movie in chunks
block_size = np.int(np.ceil((end_frame - start_frame) / nb_jobs))
# We force block_size to be a multiple of batch size
block_size = int(np.floor(block_size / batch_size) * batch_size)
jobdir = os.path.join(
output_folder,
"tmp_" + os.path.splitext(os.path.basename(model_file))[0]
)
try:
os.mkdir(jobdir)
except Exception:
print("Folder already created")
files = glob.glob(os.path.join(jobdir, "*"))
for f in files:
os.remove(f)
python_file = (r"/home/jeromel/Documents/Projects/Deep2P/repos/" +
r"deepinterpolation/examples/paper_generation_code/" +
r"cluster_lib/single_ephys_section_inferrence.py")
list_files_check = []
for index, local_start_frame in enumerate(
np.arange(start_frame, end_frame, block_size)
):
local_path = os.path.join(jobdir, "movie_" + str(index) + ".hdf5")
local_end_frame = np.min([end_frame,
local_start_frame + block_size - 1])
job_settings = {
"queue": "braintv",
"mem": "180g",
"walltime": "48:00:00",
"ppn": 16,
}
out_file = os.path.join(jobdir, "$PBS_JOBID.out")
list_files_check.append(local_path + ".done")
job_settings.update(
{
"outfile": out_file,
"errfile": os.path.join(jobdir, "$PBS_JOBID.err"),
"email": "*****@*****.**",
"email_options": "a",
}
)
arg_to_pass = [
"--movie_path "
+ dat_file
+ " --frame_start "
+ str(local_start_frame)
+ " --frame_end "
+ str(local_end_frame)
+ " --output_file "
+ local_path
+ " --model_file "
+ new_model_file
+ " --batch_size "
+ str(batch_size)
+ " --pre_post_frame "
+ str(pre_post_frame)
+ " --pre_post_omission "
+ str(pre_post_omission)
]
PythonJob(
python_file,
python_executable=(r"/allen/programs/braintv/workgroups/" +
r"nc-ophys/Jeromel/conda/tf20-env/bin/python"),
conda_env=(r"/allen/programs/braintv/workgroups/nc-ophys/" +
r"Jeromel/conda/tf20-env"),
jobname="ephys_inferrence",
python_args=arg_to_pass[0],
**job_settings
).run(dryrun=False)
# We wait for the jobs to complete
stay_in_loop = True
while stay_in_loop:
time.sleep(60)
nb_file = 0
for indiv_file in list_files_check:
if os.path.isfile(indiv_file):
nb_file += 1
if nb_file == len(list_files_check):
stay_in_loop = False
# We merge the files
output_merged = os.path.join(output_folder,
"movie_" + os.path.basename(model_file))
list_files = glob.glob(os.path.join(jobdir, "*.hdf5"))
list_files = sorted(
list_files, key=lambda x: int(x.split("movie_")[1].split(".hdf5")[0])
)
nb_frames = 0
for each_file in list_files:
with h5py.File(each_file, "r") as file_handle:
local_shape = file_handle["data"].shape
nb_frames = nb_frames + local_shape[0]
final_shape = list(local_shape)
final_shape[0] = nb_frames
global_index_frame = 0
with h5py.File(output_merged, "w") as file_handle:
dset_out = file_handle.create_dataset(
"data",
shape=final_shape,
chunks=(1, final_shape[1], final_shape[2], final_shape[3]),
dtype="float16",
)
for each_file in list_files:
with h5py.File(each_file, "r") as file_handle:
local_shape = file_handle["data"].shape
dset_out[
global_index_frame: global_index_frame + local_shape[0],
:, :, :
] = file_handle["data"][:, :, :, :]
global_index_frame += local_shape[0]
shutil.rmtree(jobdir)
job_settings = {
'queue': 'braintv',
'mem': '15g',
'walltime': '0:30:00',
'ppn': 1,
'jobdir': jobdir,
}
cache_json = {
'manifest_path':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/visual_behavior_data_manifest.csv',
'nwb_base_dir':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/nwb_files',
'analysis_files_base_dir':
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/SWDB_2019/extra_files'
}
cache = bpc.BehaviorProjectCache(cache_json)
experiment_ids = cache.manifest['ophys_experiment_id'].values
for experiment_id in experiment_ids:
PythonJob(
python_file,
python_executable='/home/nick.ponvert/anaconda3/envs/allen/bin/python',
python_args=experiment_id,
conda_env=None,
jobname='trial_response_df_{}'.format(experiment_id),
**job_settings).run(dryrun=False)
run_params["case"] = case
if net_name is not None:
run_params["net_name"] = net_name
# prepare args
params_list = list(
chain.from_iterable(
(f"--{k}", str(run_params[k])) for k in run_params))
params_string = " ".join(params_list)
# kick off HPC job
PythonJob(script,
python_executable,
conda_env=conda_env,
python_args=params_string,
jobname=job_title,
jobdir=job_dir,
**job_settings).run(dryrun=False)
def param_arr_helper(param_arr):
if param_arr is None or len(param_arr) == 0:
return None
return " ".join(str(p) for p in param_arr)
if __name__ == "__main__":
args = parser.parse_args()
# 913106887, 913106889, 913106893, 913106896]
#VisualBehaviorTask1B 7/29/19
experiment_ids = [
899085531, 901559828, 901560861, 902474078, 902487182, 902487200,
902490609, 903396212, 903403819, 903405627, 904155140, 904155155,
904161089, 905495248, 908350502, 908350518, 909090479, 910213154,
910600099, 910608808, 911146705, 911149119, 911149136, 911642167, 911642184
]
python_file = r"/home/marinag/visual_behavior_analysis/scripts/convert_level_1_to_level_2.py"
jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords2'
job_settings = {
'queue': 'braintv',
'mem': '30g',
'walltime': '5:00:00',
'ppn': 1,
'jobdir': jobdir,
}
for experiment_id in experiment_ids:
PythonJob(python_file,
python_executable=
'/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python',
python_args=experiment_id,
conda_env=None,
jobname='process_{}'.format(experiment_id),
**job_settings).run(dryrun=False)
jobdir, run_uid + os.path.basename(python_file) + "_running_terminal.txt"
)
job_settings = {
"queue": "braintv",
"mem": "250g",
"walltime": "200:00:00",
"ppn": 16,
"gpus": 1,
}
job_settings.update(
{
"outfile": os.path.join(jobdir, "$PBS_JOBID.out"),
"errfile": os.path.join(jobdir, "$PBS_JOBID.err"),
"email": "*****@*****.**",
"email_options": "a",
}
)
arg_to_pass = ""
PythonJob(
python_file,
python_executable="/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env/bin/python", # "/home/jeromel/.conda/envs/deep_work_gpu/bin/python",
conda_env="/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env", # "deep_work_gpu",
jobname="tf_" + os.path.basename(python_file),
python_args=arg_to_pass + " > " + output_terminal,
**job_settings
).run(dryrun=False)
def main(argv):
opts, args = getopt.getopt(
argv,
[],
[
"output_folder=",
"model_file=",
"start_frame=",
"end_frame=",
"pre_post_frame=",
"nb_jobs=",
"dat_file=",
"pre_post_omission=",
],
)
for opt, arg in opts:
if opt == "--output_folder":
output_folder = arg
if opt == "--model_file":
model_file = arg
if opt == "--start_frame":
start_frame = int(arg)
if opt == "--end_frame":
end_frame = int(arg)
if opt == "--pre_post_frame":
pre_post_frame = int(arg)
if opt == "--nb_jobs":
nb_jobs = int(arg)
if opt == "--dat_file":
dat_file = arg
if opt == "--pre_post_omission":
pre_post_omission = int(arg)
try:
os.mkdir(output_folder)
except:
print("Folder already created")
batch_size = 200
# We infer the movie in chunks
block_size = np.int(np.ceil((end_frame - start_frame) / nb_jobs))
# We force block_size to be a multiple of batch size
block_size = int(np.floor(block_size / batch_size) * batch_size)
jobdir = os.path.join(
output_folder, "tmp_" + os.path.splitext(os.path.basename(model_file))[0]
)
try:
os.mkdir(jobdir)
except:
print("Folder already created")
files = glob.glob(os.path.join(jobdir, "*"))
for f in files:
os.remove(f)
python_file = "/home/jeromel/Documents/Projects/Deep2P/repos/deepinterpolation/examples/cluster_lib/single_ephys_section_inferrence.py"
list_files_check = []
for index, local_start_frame in enumerate(
np.arange(start_frame, end_frame, block_size)
):
local_path = os.path.join(jobdir, "movie_" + str(index) + ".hdf5")
local_end_frame = np.min([end_frame, local_start_frame + block_size - 1])
job_settings = {
"queue": "braintv",
"mem": "180g",
"walltime": "48:00:00",
"ppn": 16,
}
out_file = os.path.join(jobdir, "$PBS_JOBID.out")
list_files_check.append(local_path + ".done")
job_settings.update(
{
"outfile": out_file,
"errfile": os.path.join(jobdir, "$PBS_JOBID.err"),
"email": "*****@*****.**",
"email_options": "a",
}
)
arg_to_pass = [
"--movie_path "
+ dat_file
+ " --frame_start "
+ str(local_start_frame)
+ " --frame_end "
+ str(local_end_frame)
+ " --output_file "
+ local_path
+ " --model_file "
+ model_file
+ " --batch_size "
+ str(batch_size)
+ " --pre_post_frame "
+ str(pre_post_frame)
+ " --pre_post_omission "
+ str(pre_post_omission)
]
PythonJob(
python_file,
python_executable="/home/jeromel/.conda/envs/deep_work2/bin/python",
conda_env="deep_work2",
jobname="movie_2p",
python_args=arg_to_pass[0],
**job_settings
).run(dryrun=False)
# We wait for the jobs to complete
stay_in_loop = True
while stay_in_loop:
time.sleep(60)
nb_file = 0
for indiv_file in list_files_check:
if os.path.isfile(indiv_file):
nb_file += 1
if nb_file == len(list_files_check):
stay_in_loop = False
# We merge the files
output_merged = os.path.join(output_folder, "movie_" + os.path.basename(model_file))
list_files = glob.glob(os.path.join(jobdir, "*.hdf5"))
list_files = sorted(
list_files, key=lambda x: int(x.split("movie_")[1].split(".hdf5")[0])
)
nb_frames = 0
for each_file in list_files:
with h5py.File(each_file, "r") as file_handle:
local_shape = file_handle["data"].shape
nb_frames = nb_frames + local_shape[0]
final_shape = list(local_shape)
final_shape[0] = nb_frames
global_index_frame = 0
with h5py.File(output_merged, "w") as file_handle:
dset_out = file_handle.create_dataset(
"data",
shape=final_shape,
chunks=(1, final_shape[1], final_shape[2], final_shape[3]),
dtype="float16",
)
for each_file in list_files:
with h5py.File(each_file, "r") as file_handle:
local_shape = file_handle["data"].shape
dset_out[
global_index_frame : global_index_frame + local_shape[0], :, :, :
] = file_handle["data"][:, :, :, :]
global_index_frame += local_shape[0]
shutil.rmtree(jobdir)
'email_options': 'a'
})
jobname = 'sg_mod_aapc'
units_savename = os.path.join(units_savedir,expt_id + '_units_df.pkl')
units_df = pd.read_pickle(units_savename)
jobcount = 0
for cstr in cstr_list:
visp_units = units_df[units_df.ecephys_structure_acronym ==cstr].index.values
no_of_jobs = len(visp_units) * len(stim_scale_list)
while (jobcount + no_of_jobs) >= 8000:
time.sleep(30)
out = subprocess.Popen(["qstat", "-u rami"], stdout=subprocess.PIPE)
out2 = subprocess.Popen(["wc", "-l"], stdin=out.stdout,
stdout=subprocess.PIPE).communicate()[0].strip('\n')
jobcount = int(float(out2))
if len(visp_units) > 0:
jobcount = jobcount + no_of_jobs
for unit_id in visp_units:
for stim_scale in stim_scale_list:
job_settings['jobname'] = '%s:%s:%s:%s:%s' % (jobname, expt_id, cstr, unit_id, stim_scale)
PythonJob(os.path.join(basedir,'sandbox_pop_coupling_with_modules.py'),
python_args='--cstr=%s --expt_id=%s --unit_id=%s --prs_yaml_fname=%s --stim_scale=%s' % (cstr, expt_id, unit_id, prs_yaml_fname, stim_scale),
conda_env=conda_env,
python_path=None,
**job_settings).run(dryrun=dryrun)
def main(net_name, config_groups, scheme, resume, lr, lr_step_size, lr_gamma,
batch_size, dataset):
job_title = "train_net"
# script, run_params and job_settings
with open("job_params.json", "r") as json_file:
job_params = json.load(json_file)
with open("net_configs.json", "r") as json_file:
net_configs = json.load(json_file)
# get cases in current groups
cases = set()
for group in config_groups:
configs = net_configs[group]
for case in configs.keys():
cases.add(case)
job_params = job_params[job_title]
script = job_params["script"]
# update any run params
run_params = job_params["run_params"]
run_params["net_name"] = net_name
run_params["scheme"] = scheme
run_params["dataset"] = dataset
if lr is not None:
run_params["lr"] = lr
run_params[
"lr_step_size"] = lr_step_size if lr_step_size is not None else run_params[
"lr_step_size"]
run_params["lr_gamma"] = lr_gamma if lr_gamma is not None else run_params[
"lr_gamma"]
run_params[
"batch_size"] = batch_size if batch_size is not None else run_params[
"batch_size"]
job_settings = job_params["job_settings"]
# set to avoid submitting jobs for the same net twice
net_filepaths = set()
# walk dir looking for nets to train
net_dir = os.path.join(run_params["data_dir"],
f"nets/{dataset}/{net_name}")
for root, dirs, files in os.walk(net_dir):
# only interested in locations files (nets) are saved
if len(files) <= 0:
continue
slugs = root.split("/")
# only interested in the given dataset
if not dataset in slugs:
continue
# only interested in the given training scheme
if not scheme in slugs:
continue
# only interested in the given groups...
if not any(g in slugs for g in config_groups):
continue
# ...and their cases
if not any(c in slugs for c in cases):
continue
# start from first or last epoch
if resume:
net_filename = get_last_epoch(files)
print(f"Submitting job to resume training of {net_filename}.")
else:
net_filename = get_first_epoch(files)
print(f"Job will begin from initial snapshot {net_filename}.")
# and add it to the training job set
net_filepath = os.path.join(root, net_filename)
net_filepaths.add(net_filepath)
# loop over set, submitting jobs
for net_filepath in net_filepaths:
# update param
run_params["net_filepath"] = net_filepath
# prepare args
params_list = list(
chain.from_iterable(
(f"--{k}", str(run_params[k])) for k in run_params))
params_string = " ".join(params_list)
# kick off HPC job
PythonJob(script,
python_executable,
conda_env=conda_env,
python_args=params_string,
jobname=job_title + f" {net_filepath}",
jobdir=job_dir,
**job_settings).run(dryrun=False)
from pbstools import PythonJob # flake8: noqa: E999
import visual_behavior.data_access.loading as loading
python_file = r"/home/marinag/visual_behavior_analysis/scripts/concatenate_stimulus_presentations.py"
jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords'
job_settings = {
'queue': 'braintv',
'mem': '50g',
'walltime': '10:00:00',
'ppn': 1,
'jobdir': jobdir,
}
experiments_table = loading.get_filtered_ophys_experiment_table()
for project_code in experiments_table.project_code.unique():
for session_number in experiments_table.session_number.unique():
PythonJob(
python_file,
python_executable=
'/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python',
python_args=[project_code, session_number],
conda_env=None,
jobname='stimulus_presentations_' + project_code + '_' +
str(session_number),
**job_settings).run(dryrun=False)
import sys
import platform
if platform.system() == 'Linux':
# sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/Doug/pbstools')
sys.path.append('/allen/programs/braintv/workgroups/nc-ophys/nick.ponvert/src/pbstools')
from pbstools import PythonJob # flake8: noqa: E999
python_file = r"/home/marinag/visual_behavior_analysis/scripts/create_multi_session_mean_df.py"
jobdir = '/allen/programs/braintv/workgroups/nc-ophys/Marina/ClusterJobs/JobRecords2'
job_settings = {'queue': 'braintv',
'mem': '200g',
'walltime': '40:00:00',
'ppn': 1,
'jobdir': jobdir,
}
PythonJob(
python_file,
python_executable='/home/marinag/anaconda2/envs/visual_behavior_sdk/bin/python',
python_args=None,
conda_env=None,
jobname='process_multi_session_df',
**job_settings
).run(dryrun=False)
"mem": "180g",
"walltime": "12:00:00",
"ppn": 16,
}
out_file = os.path.join(jobdir, "$PBS_JOBID.out")
job_settings.update({
"outfile": out_file,
"errfile": os.path.join(jobdir, "$PBS_JOBID.err"),
"email": "*****@*****.**",
"email_options": "a",
})
arg_to_pass = [
"--json_path " + json_path + " --output_file " + local_path +
" --model_file " + os.path.join(access_to_models, model_file) +
" --batch_size " + str(batch_size) + " --pre_frame " +
str(pre_frame) + " --post_frame " + str(post_frame)
]
PythonJob(
python_file,
python_executable=
"/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env/bin/python",
conda_env=
"/allen/programs/braintv/workgroups/nc-ophys/Jeromel/conda/tf20-env",
jobname="movie_2p",
python_args=arg_to_pass[0],
**job_settings).run(dryrun=False)
job_settings = {
"queue": "braintv",
"mem": "250g",
"walltime": "24:00:00",
"ppn": 16,
}
job_settings.update({
"outfile": os.path.join(jobdir, "$PBS_JOBID.out"),
"errfile": os.path.join(jobdir, "$PBS_JOBID.err"),
"email": "*****@*****.**",
"email_options": "a",
})
arg_to_pass = (" --dat_file " + dat_file + " --output_folder " +
output_folder + " --model_file " + model_file)
arg_to_pass += (" --start_frame " + str(start_frame) + " --end_frame " +
str(end_frame) + " --pre_post_frame " +
str(pre_post_frame))
arg_to_pass += (" --nb_jobs " + str(nb_jobs) + " --pre_post_omission " +
str(pre_post_omission))
PythonJob(
python_file,
python_executable="/home/jeromel/.conda/envs/deep_work2/bin/python",
conda_env="deep_work2",
jobname="movie_2p",
python_args=arg_to_pass + " > " + output_terminal,
**job_settings).run(dryrun=False)
def main(dataset, net_names, schemes, config_groups, find_lr_avg):
job_title = "gen_nets"
# script, run_params and job_settings
with open("job_params.json", "r") as json_file:
job_params = json.load(json_file)
with open("net_configs.json", "r") as json_file:
net_configs = json.load(json_file)
job_params = job_params[job_title]
script = job_params["script"]
run_params = job_params["run_params"]
run_params["dataset"] = dataset
job_settings = job_params["job_settings"]
# kick off job for each net configuration
for net_name in net_names:
for scheme in schemes:
for group in config_groups:
# get net configs in group
configs = net_configs[group]
for case in configs.keys():
# get net config details
config = configs[case]
# update params for this net config
run_params["net_name"] = net_name
run_params["scheme"] = scheme
run_params["group"] = group
run_params["case"] = case
if config.get("conv_layers") is not None:
run_params["conv_layers"] = config["conv_layers"]
if config.get("fc_layers") is not None:
run_params["fc_layers"] = config["fc_layers"]
if config.get("default_fn") is not None:
run_params["default_fn"] = config["default_fn"]
act_fns = config["act_fns"]
run_params["act_fns"] = param_arr_helper(act_fns)
if config.get("act_fn_params") is not None:
run_params["act_fn_params"] = param_arr_helper(config.get("act_fn_params"))
# default repeating each fn once
if config.get("n_repeat") is None:
n_repeat = [1] * len(act_fns)
else:
n_repeat = config.get("n_repeat")
run_params["n_repeat"] = param_arr_helper(n_repeat)
# prepare args
params_list = list(chain.from_iterable((f"--{k}", str(run_params[k])) for k in run_params))
pretrained = config.get("pretrained")
if pretrained:
params_list.append("--pretrained")
spatial = config.get("spatial")
if spatial:
params_list.append("--spatial")
cfg_find_lr_avg = config.get("find_lr_avg")
if cfg_find_lr_avg or find_lr_avg:
params_list.append("--find_lr_avg")
params_string = " ".join(params_list)
# kick off HPC job
PythonJob(
script,
python_executable,
conda_env = conda_env,
python_args = params_string,
jobname = job_title + f" c-{case}",
jobdir = job_dir,
**job_settings
).run(dryrun=False)
def main(net_name, cases, scheme, dataset, final):
job_title = "save_net_activations"
# script, run_params and job_settings
with open("job_params.json", "r") as json_file:
job_params = json.load(json_file)
with open("net_configs.json", "r") as json_file:
net_configs = json.load(json_file)
job_params = job_params[job_title]
script = job_params["script"]
# update any run params
run_params = job_params["run_params"]
run_params["net_name"] = net_name
run_params["dataset"] = dataset
job_settings = job_params["job_settings"]
# set to avoid submitting jobs for the same net twice
net_filepaths = set()
# walk dir looking for nets to train
net_dir = os.path.join(run_params["data_dir"],
f"nets/{dataset}/{net_name}/{scheme}")
for root, dirs, files in os.walk(net_dir):
# only interested in locations files (nets) are saved
if len(files) <= 0:
continue
slugs = root.split("/")
# only interested in the given dataset
if not dataset in slugs:
continue
# only interested in the given training scheme
if not scheme in slugs:
continue
# only interested in the given cases (unless [])
if len(cases) > 0 and not any(c in slugs for c in cases):
continue
# start from first or last epoch
if final:
net_filename = get_last_epoch(files)
print(
f"Submitting job to save activation output of {net_filename}.")
else:
net_filename = get_first_epoch(files)
print(
f"Job will save activations of initial snapshot {net_filename}."
)
# and add it to the training job set
net_filepath = os.path.join(root, net_filename)
net_filepaths.add(net_filepath)
# loop over set, submitting jobs
for net_filepath in net_filepaths:
# update param
run_params["net_filepath"] = net_filepath
# prepare args
params_list = list(
chain.from_iterable(
(f"--{k}", str(run_params[k])) for k in run_params))
params_string = " ".join(params_list)
# kick off HPC job
PythonJob(script,
python_executable,
conda_env=conda_env,
python_args=params_string,
jobname=job_title + f" {net_filepath}",
jobdir=job_dir,
**job_settings).run(dryrun=False)
