def reprocess_dl1(
input_dir,
config_file=None,
dl1_files_per_job=50,
particle=None,
prod_id=None,
batch_config=None,
offset=None,
workflow_kind="lstchain",
n_jobs_parallel=50,
):
"""
Reprocessing of existing dl1 files.
Organizes files and launches slurm jobs in two slurm arrays.
The same train/test split performed with the earlier r0 to dl1
processing is used.
Parameters
----------
input_dir : str
path to the files directory to analyse
config_file :str
Path to a configuration file. If none is given, the standard configuration of the selected pipeline is applied
particle : str
particle type (gamma/gamma_off/proton/electron). Determines output directory structure, job naming
and n_r0_files_per_dl1_job if not set explicitly.
offset : str
gamma offset
prod_id :str
Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
batch_config : dict
Dictionary containing the (full) source_environment and the slurm_account strings.
! NOTE : train_pipe AND dl1_to_dl2 **MUST** be run with the same environment.
workflow_kind: str
One of the supported pipelines. Defines the command to be run on r0 files
n_jobs_parallel: int
Number of jobs to be run at the same time per array.
dl1_files_per_job: int
Number of dl1 files to be processed per job array that was batched.
Returns
-------
jobid2log : dict
A dictionary of dictionaries containing the full log information of the script. The first `layer` contains
only the each jobid that the scripts has batched.
dict[jobid] = information
The second layer contains, organized by jobid,
- the kind of particle that corresponded to the jobid
- the command that was run to batch the job into the server
- the path to both the output and error files (job_`jobid`.o and job_`jobid`.e) that were generated
when the job was send to the cluster
dict[jobid].keys() = ['particle', 'sbatch_command', 'jobe_path', 'jobo_path']
jobids_dl1_dl1
A list of all the jobs sent by particle (including test and train set types).
"""
log.info("Starting DL1 to DL1 processing for particle {}".format(particle))
input_dir = Path(input_dir)
source_environment = batch_config["source_environment"]
slurm_account = batch_config["slurm_account"]
if workflow_kind == "lstchain":
base_cmd = f"{source_environment} lstmcpipe_lst_core_dl1ab -c {config_file} "
jobtype_id = "LST"
elif workflow_kind == "ctapipe":
base_cmd = f"{source_environment} lstmcpipe_cta_core_r0_dl1 -c {config_file} "
jobtype_id = "CTA"
else:
log.critical("Please, selected an allowed workflow kind.")
exit(-1)
job_name = {
"electron": f"e_{jobtype_id}_dl1dl1",
"gamma": f"g_{jobtype_id}_dl1dl1",
"gamma-diffuse": f"gd_{jobtype_id}_dl1dl1",
"proton": f"p_{jobtype_id}_dl1dl1",
"gamma_off0.0deg": f"g0.0_{jobtype_id}_dl1dl1",
"gamma_off0.4deg": f"g0.4_{jobtype_id}_dl1dl1",
}
log.info("Working on DL1 files in {}".format(input_dir.as_posix()))
training_input_path = input_dir / "training"
check_data_path(training_input_path)
training_list = get_input_filelist(training_input_path)
ntrain = len(training_list)
testing_input_path = input_dir / "testing"
check_data_path(testing_input_path)
testing_list = get_input_filelist(testing_input_path)
ntest = len(testing_list)
log.info("{} DL1 files in training dataset".format(ntrain))
log.info("{} DL1 files in test dataset".format(ntest))
with open("training.list", "w+") as newfile:
for f in training_list:
newfile.write(f)
newfile.write("\n")
with open("testing.list", "w+") as newfile:
for f in testing_list:
newfile.write(f)
newfile.write("\n")
if "off" in particle:
# Take out /off0.Xdeg
running_dir = (Path(input_dir.as_posix().replace(
"DL1", "running_analysis")).parent.parent / prod_id / offset)
else:
running_dir = (Path(input_dir.as_posix().replace(
"DL1", "running_analysis")).parent / prod_id)
job_logs_dir = running_dir / "job_logs"
dl1_output_dir = running_dir / "DL1"
log.info("RUNNING_DIR: {}".format(running_dir))
log.info("JOB_LOGS DIR: {}".format(job_logs_dir))
log.info("DL1 DATA DIR: {}".format(dl1_output_dir))
for directory in [running_dir, dl1_output_dir, job_logs_dir]:
check_and_make_dir_without_verification(directory)
# dumping the training and testing lists and splitting them in sub-lists for parallel jobs
# copy config into working dir
if config_file is not None:
shutil.copyfile(config_file,
running_dir.joinpath(Path(config_file).name))
# save file lists into logs
shutil.move("testing.list", running_dir.joinpath("testing.list"))
shutil.move("training.list", running_dir.joinpath("training.list"))
jobid2log, jobids_dl1_dl1 = submit_dl1_jobs(
base_cmd=base_cmd,
file_lists={
"testing": testing_list,
"training": training_list
},
particle=particle,
job_name=job_name[particle],
dl1_files_per_batched_job=dl1_files_per_job,
running_dir=running_dir,
job_logs_dir=job_logs_dir,
n_jobs_parallel=n_jobs_parallel,
slurm_account=slurm_account)
# return it log dictionary
return jobid2log, jobids_dl1_dl1
def submit_dl1_jobs(base_cmd, file_lists, particle, job_name,
dl1_files_per_batched_job, running_dir, job_logs_dir,
n_jobs_parallel, slurm_account):
jobid2log = {}
jobids_dl1 = []
# types should be training and testing
for set_type, list_type in file_lists.items():
log.debug("Generating list for {} step".format(set_type))
dir_lists = running_dir.joinpath("file_lists_" + set_type)
output_dir = running_dir.joinpath("DL1", set_type)
check_and_make_dir_without_verification(dir_lists)
check_and_make_dir_without_verification(output_dir)
log.info("output dir: {}".format(output_dir))
number_of_sublists = len(list_type) // dl1_files_per_batched_job + int(
len(list_type) % dl1_files_per_batched_job > 0)
for i in range(number_of_sublists):
output_file = dir_lists.joinpath("{}_{}.list".format(
set_type, i)).as_posix()
with open(output_file, "w+") as out:
for line in list_type[i * dl1_files_per_batched_job:
dl1_files_per_batched_job * (i + 1)]:
out.write(line)
out.write("\n")
log.info(f"{number_of_sublists} files generated for {set_type} list")
files = [f.as_posix() for f in Path(dir_lists).glob("*")]
slurm_options = {
"output":
job_logs_dir.joinpath(
f"job_%A_%a_{'train' if set_type=='training' else 'test'}.o").
as_posix(),
"error":
job_logs_dir.joinpath(
f"job_%A_%a_{'train' if set_type=='training' else 'test'}.o").
as_posix(),
"array":
f"0-{len(files)-1}%{n_jobs_parallel}",
"job-name":
f"{job_name}"
}
if particle == "proton":
slurm_options.update({"partition": "long"})
else:
# Computing time sometimes depends on the machine.
# e and g_0.X should be working on short, but sometimes runs on timeout.
slurm_options.update({"partition": "long"})
# `sbatch -A` is the --account slurm argument
if slurm_account != '':
slurm_options.update({"account": slurm_account})
# start 1 jobarray with all files included. The job selects its file based on its task id
cmd = f'{base_cmd} -f {" ".join(files)} --output_dir {output_dir}'
slurm_cmd = "sbatch --parsable "
for key, value in slurm_options.items():
slurm_cmd += f"--{key} {value} "
slurm_cmd += f'--wrap="{cmd}"'
log.debug(f"Slurm command to start the jobs: {slurm_cmd}")
jobid = os.popen(slurm_cmd).read().strip("\n")
log.debug(f"Submitted batch job {jobid}")
jobids_dl1.append(jobid)
jobid2log[jobid] = {
"particle": particle,
"set_type": set_type,
"jobe_path": slurm_options["error"],
"jobo_path": slurm_options["output"],
"sbatch_command": slurm_cmd
}
return jobid2log, jobids_dl1
def r0_to_dl1(
input_dir,
config_file=None,
train_test_ratio=0.5,
rng=None,
dl1_files_per_job=None,
particle=None,
prod_id=None,
batch_config=None,
offset=None,
workflow_kind="lstchain",
keep_rta_file=False,
n_jobs_parallel=50,
debug_mode=False,
):
"""
R0 to DL1 MC onsite conversion.
Organizes files and launches slurm jobs in two slurm arrays.
Parameters
----------
input_dir : str
path to the files directory to analyse
config_file :str
Path to a configuration file. If none is given, the standard configuration of the selected pipeline is applied
train_test_ratio: float
Ratio of training data. Default = 0.5
rng : int
Random seed for random processes. Default = 42
dl1_files_per_job : int
Number of r0 files processed by each r0_to_dl1 batched stage. If set to None (Default), see below the `usual
production` case.n_r0_files_per_dl1_job
If the number of r0 files found in `input_dir` is less than 100, it is consider to be a test on a small
production. Therefore, the number of r0 files treated per batched stage (r0_to_dl1 or dl1ab) will be set to 10.
Usual productions have >=1000 r0 files, in this case, the number of files per job (r0_to_dl1 or dl1ab) will be
fixed to 50 (in case of gamma and electrons), and 100 for protons. Because of this protons end up in the
long queue and other particles are submitted to the short queue.
particle : str
particle type (gamma/gamma_off/proton/electron). Determines output directory structure, job naming
and n_r0_files_per_dl1_job if not set explicitly.
offset : str
gamma offset
prod_id :str
Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
batch_config : dict
Dictionary containing the full (source + env) source_environment and the slurm_account strings.
! NOTE : train_pipe AND dl1_to_dl2 **MUST** be run with the same environment.
workflow_kind: str
One of the supported pipelines. Defines the command to be run on r0 files
n_jobs_parallel: int
Number of jobs to be run at the same time per array.
# HIPERTA ARGUMENTS
keep_rta_file : bool
Flag to indicate whether to keep (keep_rta_file = True) or remove (keep_rta_file = Flase ) the
`dl1v06_reorganized_*.h5` output file (hiperta_r0_dl1 and re-organiser stages).
Argument to be passed to the hiperta_r0_to_dl1lstchain script.
debug_mode : bool
Flag to activate debug_mode. Only compatible with `hiperta` workflow kind, i.e.,
HiPeRTA functionality. DEFAULT=False.
Returns
-------
jobid2log : dict
A dictionary of dictionaries containing the full log information of the script. The first `layer` contains
only the each jobid that the scripts has batched.
dict[jobid] = information
The second layer contains, organized by jobid,
- the kind of particle that corresponded to the jobid
- the command that was run to batch the job into the server
- the path to both the output and error files (job_`jobid`.o and job_`jobid`.e) that were generated
when the job was send to the cluster
dict[jobid].keys() = ['particle', 'sbatch_command', 'jobe_path', 'jobo_path']
jobids_r0_dl1
A list of all the jobs sent by particle (including test and train set types).
"""
log.info("Starting R0 to DL1 processing for particle {}".format(particle))
train_test_ratio = float(train_test_ratio)
source_environment = batch_config["source_environment"]
slurm_account = batch_config["slurm_account"]
if workflow_kind == "lstchain":
base_cmd = f"{source_environment} lstmcpipe_lst_core_r0_dl1 -c {config_file} "
jobtype_id = "LST"
elif workflow_kind == "ctapipe":
base_cmd = f"{source_environment} lstmcpipe_cta_core_r0_dl1 -c {config_file} "
jobtype_id = "CTA"
elif workflow_kind == "hiperta":
base_cmd = (
f"{source_environment} lstmcpipe_rta_core_r0_dl1 -c {config_file} "
)
if keep_rta_file:
base_cmd += " --keep_rta_file"
if debug_mode:
base_cmd += " --debug_mode"
jobtype_id = "RTA"
else:
base_cmd = ''
jobtype_id = ''
log.critical("Please, select an allowed workflow kind.")
exit(-1)
job_name = {
"electron": f"e_{jobtype_id}_r0dl1",
"gamma": f"g_{jobtype_id}_r0dl1",
"gamma-diffuse": f"gd_{jobtype_id}_r0dl1",
"proton": f"p_{jobtype_id}_r0dl1",
"gamma_off0.0deg": f"g0.0_{jobtype_id}_r0dl1",
"gamma_off0.4deg": f"g0.4_{jobtype_id}_r0dl1",
}
log.info("Working on DL0 files in {}".format(input_dir))
check_data_path(input_dir)
raw_files_list = get_input_filelist(input_dir)
if len(raw_files_list) < 100:
dl1_files_per_job = 10
else:
if "gamma" in input_dir:
dl1_files_per_job = 25
elif "gamma-diffuse" in input_dir or "electron" in input_dir:
dl1_files_per_job = 50
elif "proton" in input_dir:
dl1_files_per_job = 50
else:
dl1_files_per_job = 50
if rng is None:
rng = default_rng()
rng.shuffle(raw_files_list)
number_files = len(raw_files_list)
ntrain = int(number_files * train_test_ratio)
ntest = number_files - ntrain
training_list = raw_files_list[:ntrain]
testing_list = raw_files_list[ntrain:]
log.info("{} raw R0 files".format(number_files))
log.info("{} R0 files in training dataset".format(ntrain))
log.info("{} R0 files in test dataset".format(ntest))
with open("training.list", "w+") as newfile:
for f in training_list:
newfile.write(f)
newfile.write("\n")
with open("testing.list", "w+") as newfile:
for f in testing_list:
newfile.write(f)
newfile.write("\n")
if "off" in particle:
# Take out /off0.Xdeg
running_dir = (
Path(str(input_dir).replace("DL0", "running_analysis")).parent /
prod_id / offset)
else:
running_dir = (
Path(str(input_dir).replace("DL0", "running_analysis")) / prod_id)
job_logs_dir = running_dir.joinpath("job_logs")
dl1_output_dir = running_dir.joinpath("DL1")
log.info("RUNNING_DIR: {}".format(running_dir))
log.info("JOB_LOGS DIR: {}".format(job_logs_dir))
log.info("DL1 DATA DIR: {}".format(dl1_output_dir))
for directory in [running_dir, dl1_output_dir, job_logs_dir]:
check_and_make_dir_without_verification(directory)
# dumping the training and testing lists and splitting them in sub-lists for parallel jobs
jobid2log, jobids_r0_dl1 = submit_dl1_jobs(
base_cmd=base_cmd,
file_lists={
"testing": testing_list,
"training": training_list
},
particle=particle,
job_name=job_name[particle],
dl1_files_per_batched_job=dl1_files_per_job,
running_dir=running_dir,
job_logs_dir=job_logs_dir,
n_jobs_parallel=n_jobs_parallel,
slurm_account=slurm_account)
# copy config into working dir
if config_file is not None:
shutil.copyfile(config_file,
running_dir.joinpath(Path(config_file).name))
# save file lists into logs
shutil.move("testing.list", running_dir.joinpath("testing.list"))
shutil.move("training.list", running_dir.joinpath("training.list"))
# return it log dictionary
return jobid2log, jobids_r0_dl1
def dl2_to_irfs(
dl2_directory,
config_file,
log_from_dl1_dl2,
irf_point_like=True,
irf_gamma_offset="off0.0deg",
batch_configuration="",
wait_jobs_dl1dl2=None,
prod_id=None,
):
"""
Batches interactively the lstchain `lstchain_create_irf_files` entry point.
Parameters
----------
dl2_directory: str
Base path to DL2 directory, (not formatted with the particle type yet).
config_file: str
Path to a configuration file. If none is given, a standard configuration is applied
irf_point_like: bool
MC prod configuration argument to create IRFs: {True: gamma, False: gamma-diffuse}.
irf_gamma_offset: str
MC prod configuration argument to create IRFs: off0.0deg (for ON/OFF obs) or off0.4deg (for wobble obs).
batch_configuration : dict
Dictionary containing the (full) source_environment and the slurm_account strings to be passed to the
sbatch commands
log_from_dl1_dl2: dict
Dictionary with dl2 output path. Files are not yet here, but path and full name are needed to batch the job.
wait_jobs_dl1dl2: str
Comma separated string with the job ids of previous stages (dl1_to_dl2 stage) to be passed as dependencies to
the create_irfs_files job to be batched.
prod_id: str
prod_id defined within config_MC_prod.yml file
Returns
-------
log_dl2_to_irfs: dict
Dictionary-wise log containing {'job_id': 'batched_cmd'} items
list_job_id_dl2_irfs: str
Job-ids of the batched job to be passed to the last (MC prod check) stage of the workflow.
"""
source_env = batch_configuration["source_environment"]
slurm_account = batch_configuration["slurm_account"]
allowed_gamma_off = ["off0.0deg", "off0.4deg"]
if irf_gamma_offset not in allowed_gamma_off:
log.info(
f'Please select a valid gamma_offset to compute the IRFS: {" or ".join(allowed_gamma_off)}'
)
exit(-1)
if irf_point_like:
output_irfs_dir = os.path.join(
dl2_directory.replace("/DL2/", "/IRF/").replace("/{}/", "/"),
irf_gamma_offset,
)
else:
output_irfs_dir = os.path.join(
dl2_directory.replace("/DL2/", "/IRF/").replace("/{}/", "/"),
"diffuse")
log_dl2_to_irfs = {}
list_job_id_dl2_irfs = []
if not log_from_dl1_dl2: # Empty dict and thus no dl2 path files
dl2_particle_paths = check_dl2_files(dl2_directory, irf_point_like,
irf_gamma_offset)
# Comprehension list to find gamma or gamma-diffuse
gamma_kind = [
g for g in dl2_particle_paths.keys() if g.startswith("gamma")
][0]
gamma_file = dl2_particle_paths[gamma_kind]
proton_file = dl2_particle_paths["proton"]
electron_file = dl2_particle_paths["electron"]
irf_kind = gamma_kind
else:
proton_file = log_from_dl1_dl2["proton"]["dl2_test_path"]
electron_file = log_from_dl1_dl2["electron"]["dl2_test_path"]
if irf_point_like and irf_gamma_offset == "off0.0deg":
gamma_file = log_from_dl1_dl2["gamma_off0.0deg"]["dl2_test_path"]
irf_kind = irf_gamma_offset
elif irf_point_like and irf_gamma_offset == "off0.4deg":
gamma_file = log_from_dl1_dl2["gamma_off0.4deg"]["dl2_test_path"]
irf_kind = irf_gamma_offset
else:
gamma_file = log_from_dl1_dl2["gamma-diffuse"]["dl2_test_path"]
irf_kind = "diffuse"
if irf_point_like:
point_like = "--point-like"
else:
point_like = ""
# Final outfile name with IRF kind
if prod_id is None:
output_filename_irf = os.path.join(output_irfs_dir, "irf.fits.gz")
else:
if irf_point_like:
output_filename_irf = os.path.join(
output_irfs_dir,
"irf_" + prod_id.replace(".", "") +
f'_gamma_point-like_{irf_gamma_offset.replace(".", "")}.fits.gz',
)
else:
output_filename_irf = os.path.join(
output_irfs_dir,
"irf_" + prod_id.replace(".", "") + "_gamma_diffuse.fits.gz",
)
cmd = (
f"lstchain_create_irf_files {point_like} -g {gamma_file} -p {proton_file} -e {electron_file}"
f" -o {output_filename_irf}")
if config_file:
cmd += f" --config={config_file}"
# TODO dry-run option ?
# if dry_run:
# print(cmd)
log.info(f"Output dir IRF {irf_kind}: {output_irfs_dir}")
check_and_make_dir_without_verification(output_irfs_dir)
jobe = os.path.join(output_irfs_dir, f"job_dl2_to_irfs_gamma_{irf_kind}.e")
jobo = os.path.join(output_irfs_dir, f"job_dl2_to_irfs_gamma_{irf_kind}.o")
batch_cmd = "sbatch --parsable -p short"
if slurm_account != "":
batch_cmd += f" -A {slurm_account}"
batch_cmd += (f" --dependency=afterok:{wait_jobs_dl1dl2} -J IRF_{irf_kind}"
f' -e {jobe} -o {jobo} --wrap="{source_env} {cmd}"')
job_id_dl2_irfs = os.popen(batch_cmd).read().strip("\n")
log_dl2_to_irfs[job_id_dl2_irfs] = batch_cmd
list_job_id_dl2_irfs.append(job_id_dl2_irfs)
# Copy config into working dir
if config_file:
shutil.copyfile(
config_file,
os.path.join(output_irfs_dir, os.path.basename(config_file)))
list_job_id_dl2_irfs = ",".join(list_job_id_dl2_irfs)
return log_dl2_to_irfs, list_job_id_dl2_irfs
def dl1_to_dl2(
input_dir,
path_models,
config_file,
particle,
wait_jobid_train_pipe=None,
wait_jobids_merge=None,
dictionary_with_dl1_paths=None,
batch_configuration='',
):
"""
Convert onsite files from dl1 to dl2
Parameters
----------
input_dir : str
DL1 base path
path_models : str
Path to the trained models
config_file : str
Path to a configuration file. If none is given, a standard configuration is applied
particle : str
Particle to which apply the dl1_to_dl2 stage
wait_jobid_train_pipe : str
Comma-separated string with the batched jobid from the train stage to indicate the
dependencies of the current job to be batched
wait_jobids_merge : str
string with merge_and_copy jobids
dictionary_with_dl1_paths : dict
Dictionary with 'particles' as keys containing final output filenames of dl1 files.
batch_configuration : dict
Dictionary containing the (full) source_environment and the slurm_account strings to be passed to the
sbatch commands
! NOTE : train_pipe AND dl1_to_dl2 MUST BE RUN WITH THE SAME ENVIRONMENT
Returns
-------
log_dl1_to_dl2 : dict
dictionary of dictionaries containing the jobid of the batched job as key and the run command (the
lstchain_mc_dl1_to_dl2 command with all its corresponding arguments) as value.
jobid_dl1_to_dl2 : str
jobid of the batched job to be send (for dependencies purposes) to the next stage of the
workflow (dl2_to_irfs)
"""
source_environment = batch_configuration["source_environment"]
slurm_account = batch_configuration["slurm_account"]
output_dir = input_dir.replace("DL1", "DL2")
log.info("Working on DL1 files in {}".format(input_dir))
check_and_make_dir_without_verification(output_dir)
log.info("Output dir {}: {}".format(particle, output_dir))
log_dl1_to_dl2 = {particle: {}}
# path to dl1 files by particle type
file_list = [
dictionary_with_dl1_paths[particle]["testing"]
["test_path_and_outname_dl1"]
]
return_jobids = []
if wait_jobid_train_pipe == "":
wait_jobs = wait_jobids_merge
elif wait_jobids_merge == "":
wait_jobs = wait_jobid_train_pipe
elif wait_jobids_merge == "" and wait_jobid_train_pipe == "":
wait_jobs = ""
else:
wait_jobs = ",".join([wait_jobid_train_pipe, wait_jobids_merge])
job_name = {
"electron": "dl1-2_e",
"gamma": "dl1-2_g",
"gamma-diffuse": "dl1-2_gd",
"proton": "dl1-2_p",
"gamma_off0.0deg": "dl1-2_g.0",
"gamma_off0.4deg": "dl1-2_g.4",
}
for file in file_list:
cmd = f"{source_environment} lstchain_dl1_to_dl2 -f {file} -p {path_models}" \
f" -o {output_dir}"
if config_file is not None:
cmd += f" -c {config_file}"
# TODO dry-run option ?
# if dry_run:
# print(cmd)
# Only applying dl1_to_dl2 stage to test files
if "testing" in file:
ftype = "test"
else:
ftype = "-"
jobe = os.path.join(output_dir, f"dl1_dl2_{particle}_{ftype}job.e")
jobo = os.path.join(output_dir, f"dl1_dl2_{particle}_{ftype}job.o")
# sbatch --parsable --dependency=afterok:{wait_ids_proton_and_gammas} --wrap="{cmd}"
batch_cmd = "sbatch --parsable -p short --mem=32G"
if slurm_account != "":
batch_cmd += f" -A {slurm_account}"
batch_cmd += (
f" --dependency=afterok:{wait_jobs} -J {job_name[particle]} -e {jobe}"
f' -o {jobo} --wrap="{cmd}"')
# Batch the job at La Palma
jobid_dl1_to_dl2 = os.popen(batch_cmd).read().strip("\n")
log_dl1_to_dl2[particle][jobid_dl1_to_dl2] = batch_cmd
if "testing" in file:
log_dl1_to_dl2[particle]["dl2_test_path"] = file.replace(
"/DL1/", "/DL2/").replace("dl1_", "dl2_", 1)
return_jobids.append(jobid_dl1_to_dl2)
if config_file is not None:
shutil.copyfile(
config_file, os.path.join(output_dir,
os.path.basename(config_file)))
return_jobids = ",".join(return_jobids)
return log_dl1_to_dl2, return_jobids
def main():
args = parser.parse_args()
# check_and_make_dir(args.dest) CANNOT be used because if demands user interaction.
check_and_make_dir_without_verification(args.dest)
move_dir_content(args.source, args.dest)
def train_pipe(
gamma_dl1_train_file,
proton_dl1_train_file,
config_file=None,
batch_configuration='',
wait_ids_proton_and_gammas=None,
):
"""
Train RF from MC DL1 data (onsite LaPalma cluster)
Parameters
----------
gamma_dl1_train_file: str
path to the gamma file
proton_dl1_train_file: str
path to the proton file
config_file: str
Path to a configuration file. If none is given, a standard configuration is applied
batch_configuration : dict
Dictionary containing the (full) source_environment and the slurm_account strings to be passed to the
sbatch commands
! NOTE : train_pipe AND dl1_to_dl2 MUST BE RUN WITH THE SAME ENVIRONMENT
wait_ids_proton_and_gammas : str
a string (of chained jobids separated by ',' and without spaces between each element), to indicate the
dependencies of the job to be batched
Returns
-------
log_train : dict
dictionary containing the jobid of the batched job as key and the run command (the lstchain_mc_train
command with all its corresponding arguments) as value.
jobid_train : str
jobid of the batched job to be send (for dependencies purposes) to the next stage of the
workflow (onsite_mc_dl1_to_dl2)
models_dir : str
Path with the directory where the models are stored
"""
log_train = {}
source_environment = batch_configuration["source_environment"]
slurm_account = batch_configuration["slurm_account"]
dl1_proton_dir = os.path.dirname(os.path.abspath(proton_dl1_train_file))
# check if it path follows the established paths (lstchain-like) or not (rta-like) ##
if dl1_proton_dir.find("/mc/DL1/") > 0:
models_dir = dl1_proton_dir.replace("/mc/DL1", "/models")
else:
models_dir = dl1_proton_dir.replace("/DL1", "/models")
models_dir = models_dir.replace("/proton/", "/")
log.info("Models will be placed in {}".format(models_dir))
check_and_make_dir_without_verification(models_dir)
cmd = ""
if source_environment is not None:
cmd += source_environment
cmd += (
f" lstchain_mc_trainpipe --fg {os.path.abspath(gamma_dl1_train_file)}"
f" --fp {os.path.abspath(proton_dl1_train_file)} -o {models_dir}")
if config_file is not None:
cmd = cmd + " -c {}".format(config_file)
jobo = os.path.join(models_dir, "train_job.o")
jobe = os.path.join(models_dir, "train_job.e")
# TODO dry-run option ?
# if dry_run:
# print(cmd)
# 'sbatch --parsable --dependency=afterok:{wait_ids_proton_and_gammas} -e {jobe} -o {jobo} --wrap="{base_cmd}"'
batch_cmd = "sbatch --parsable -p long --mem=32G"
if slurm_account != "":
batch_cmd += f" -A {slurm_account}"
if wait_ids_proton_and_gammas != "":
batch_cmd += " --dependency=afterok:" + wait_ids_proton_and_gammas
batch_cmd += f' -J train_pipe -e {jobe} -o {jobo} --wrap="{cmd}" '
jobid_train = os.popen(batch_cmd).read().strip("\n")
log_train[jobid_train] = batch_cmd
# copy config into working dir
if config_file is not None:
shutil.copyfile(
config_file, os.path.join(models_dir,
os.path.basename(config_file)))
return log_train, jobid_train, models_dir