def main():
print(f"\n--> Start calculating DRS4 pedestals from run {run}\n")
try:
# verify input file
file_list = sorted(Path(f"{base_dir}/R0").rglob(f'*{run}.0000*'))
if len(file_list) == 0:
print(f">>> Error: Run {run} not found under {base_dir}/R0 \n")
raise NameError()
else:
input_file = f"{file_list[0]}"
# find date
input_dir, name = os.path.split(os.path.abspath(input_file))
path, date = input_dir.rsplit('/', 1)
# verify and make output dir
output_dir = f"{base_dir}/calibration/{date}/{prod_id}"
if not os.path.exists(output_dir):
print(f"--> Create directory {output_dir}")
os.makedirs(output_dir, exist_ok=True)
# make log dir
log_dir = f"{output_dir}/log"
if not os.path.exists(log_dir):
print(f"--> Create directory {log_dir}")
os.makedirs(log_dir, exist_ok=True)
# define output file
output_file = f"{output_dir}/drs4_pedestal.Run{run}.0000.fits"
if os.path.exists(output_file):
print(f">>> Output file {output_file} exists already. ")
if query_yes_no("Do you want to remove it?"):
os.remove(output_file)
else:
print(f">>> Exit")
exit(1)
# run lstchain script
cmd = f"lstchain_data_create_drs4_pedestal_file --input-file {input_file} " \
f"--output-file {output_file} --max-events {max_events}"
os.system(cmd)
# plot and save some results
plot_file = f"{output_dir}/log/drs4_pedestal.Run{run}.0000.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
drs4.plot_pedestals(input_file,
output_file,
run,
plot_file,
tel_id=tel_id,
offset_value=300)
print("\n--> END")
except Exception as e:
print(f"\n >>> Exception: {e}")
def main():
print(f"\n--> Start calculating calibration from run {run}")
try:
# verify input file
file_list = sorted(
Path(f"{base_dir}/R0").rglob(f'*{run}.{sub_run:04d}*'))
if len(file_list) == 0:
raise IOError(f"Run {run} not found\n")
else:
input_file = file_list[0]
print(f"\n--> Input file: {input_file}")
# find date
input_dir, name = os.path.split(os.path.abspath(input_file))
path, date = input_dir.rsplit('/', 1)
# verify output dir
output_dir = f"{base_dir}/calibration/{date}/{prod_id}"
if not os.path.exists(output_dir):
raise IOError(f"Output directory {output_dir} does not exist\n")
# search the pedestal calibration file
pedestal_file = f"{output_dir}/drs4_pedestal.Run{ped_run:05d}.0000.fits"
if not os.path.exists(pedestal_file):
raise IOError(f"Pedestal file {pedestal_file} does not exist.\n ")
# search the summary file info
run_summary_path = f"{base_dir}/monitoring/RunSummary/RunSummary_{date}.ecsv"
if not os.path.exists(run_summary_path):
raise IOError(
f"Night summary file {run_summary_path} does not exist\n")
# define config file
config_file = os.path.join(
os.path.dirname(__file__),
"../../data/onsite_camera_calibration_param.json")
if not os.path.exists(config_file):
raise IOError(f"Config file {config_file} does not exists. \n")
print(f"\n--> Config file {config_file}")
#
# produce drs4 time calibration file
#
time_file = f"{output_dir}/time_calibration.Run{run:05d}.0000.h5"
print(f"\n***** PRODUCE TIME CALIBRATION FILE ***** ")
if default_time_run is 0:
print(f"\n--> PRODUCING TIME CALIBRATION in {time_file} ...")
cmd = f"lstchain_data_create_time_calibration_file --input-file {input_file} " \
f"--output-file {time_file} --config {config_file} " \
f"--run-summary-file={run_summary_path} " \
f"--pedestal-file {pedestal_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
else:
# otherwise perform a link to the default time calibration file
print(
f"\n--> PRODUCING LINK TO DEFAULT TIME CALIBRATION (run {default_time_run})"
)
file_list = sorted(
Path(f"{base_dir}/calibration/").rglob(
f'*/{prod_id}/time_calibration.Run{default_time_run}*'))
if len(file_list) == 0:
raise IOError(
f"Time calibration file for run {default_time_run} not found\n"
)
else:
time_calibration_file = file_list[0]
cmd = f"ln -sf {time_calibration_file} {time_file}"
os.system(cmd)
print(f"\n--> Time calibration file: {time_file}")
# define charge file names
print(f"\n***** PRODUCE CHARGE CALIBRATION FILE ***** ")
output_file = f"{output_dir}/calibration.Run{run:05d}.{sub_run:04d}.h5"
log_file = f"{output_dir}/log/calibration.Run{run:05d}.{sub_run:04d}.log"
print(f"\n--> Output file {output_file}")
if os.path.exists(output_file) and ff_calibration == 'yes':
if query_yes_no(
">>> Output file exists already. Do you want to remove it?"
):
os.remove(output_file)
else:
print(f"\n--> Stop")
exit(1)
print(f"\n--> Log file {log_file}")
#
# produce ff calibration file
#
if ff_calibration == 'yes':
# run lstchain script
cmd = f"lstchain_create_calibration_file " \
f"--input_file={input_file} --output_file={output_file} "\
f"--EventSource.max_events={max_events} " \
f"--EventSource.default_trigger_type=tib " \
f"--EventSource.min_flatfield_adc={min_ff} " \
f"--EventSource.max_flatfield_adc={max_ff} " \
f"--LSTEventSource.EventTimeCalculator.run_summary_path={run_summary_path} " \
f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={time_file} " \
f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={pedestal_file} " \
f"--FlatFieldCalculator.sample_size={stat_events} --PedestalCalculator.sample_size={stat_events} " \
f"--config={config_file} > {log_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
# plot and save some results
plot_file = f"{output_dir}/log/calibration.Run{run:05d}.{sub_run:04d}.pedestal.Run{ped_run:05d}.0000.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
calib.read_file(output_file, tel_id)
calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data,
run, plot_file)
print("\n--> END")
except Exception as e:
print(f"\n >>> Exception: {e}")
def main():
args = parser.parse_args()
run = args.run_number
prod_id = args.prod_version
stat_events = args.statistics
time_run = args.time_run
sys_date = args.sys_date
no_sys_correction = args.no_sys_correction
output_base_name = args.output_base_name
sub_run = args.sub_run
tel_id = args.tel_id
config_file = args.config
yes = args.yes
pro_symlink = not args.no_pro_symlink
# looks for the filter values in the database if not given
if args.filters is None:
filters = search_filter(run, args.mongodb)
else:
filters = args.filters
if filters is None:
sys.exit(f"Missing filter value for run {run}. \n")
# define the FF selection cuts
if args.min_ff is None or args.max_ff is None:
min_ff, max_ff = define_FF_selection_range(filters)
else:
min_ff, max_ff = args.min_ff, args.max_ff
print(f"\n--> Start calculating calibration from run {run}, filters {filters}")
# verify config file
if not config_file.exists():
raise IOError(f"Config file {config_file} does not exists. \n")
print(f"\n--> Config file {config_file}")
# verify input file
r0_dir = args.r0_dir or args.base_dir / 'R0'
input_file = find_r0_subrun(run, sub_run, r0_dir)
date = input_file.parent.name
print(f"\n--> Input file: {input_file}")
# verify output dir
calib_dir = args.base_dir / LEVEL_A_PIXEL_DIR
output_dir = calib_dir / "calibration" / date / prod_id
if not output_dir.exists():
print(f"--> Create directory {output_dir}")
output_dir.mkdir(parents=True, exist_ok=True)
if pro_symlink:
pro = "pro"
create_pro_symlink(output_dir)
else:
pro = prod_id
# make log dir
log_dir = output_dir / "log"
if not log_dir.exists():
print(f"--> Create directory {log_dir}")
log_dir.mkdir(parents=True, exist_ok=True)
# search the summary file info
run_summary_path = find_run_summary(date, args.base_dir)
print(f"\n--> Use run summary {run_summary_path}")
pedestal_file = find_pedestal_file(pro, args.pedestal_run, date=date, base_dir=args.base_dir)
print(f"\n--> Pedestal file: {pedestal_file}")
# search for time calibration file
time_file = find_time_calibration_file(pro, run, time_run, args.base_dir)
print(f"\n--> Time calibration file: {time_file}")
# define systematic correction file
if no_sys_correction:
systematics_file = None
else:
systematics_file = find_systematics_correction_file(pro, date, sys_date, args.base_dir)
print(f"\n--> F-factor systematics correction file: {systematics_file}")
# define charge file names
print("\n***** PRODUCE CHARGE CALIBRATION FILE ***** ")
if filters is not None:
filter_info = f"_filters_{filters}"
else:
filter_info = ""
# remember there are no systematic corrections
prefix = "no_sys_corrected_" if no_sys_correction else ""
output_name = f"{prefix}{output_base_name}{filter_info}.Run{run:05d}.{sub_run:04d}"
output_file = output_dir / f'{output_name}.h5'
print(f"\n--> Output file {output_file}")
log_file = log_dir / f"{output_name}.log"
print(f"\n--> Log file {log_file}")
if output_file.exists():
remove = False
if not yes and os.getenv('SLURM_JOB_ID') is None:
remove = query_yes_no(">>> Output file exists already. Do you want to remove it?")
if yes or remove:
os.remove(output_file)
os.remove(log_file)
else:
print("\n--> Output file exists already. Stop")
exit(1)
#
# produce ff calibration file
#
cmd = [
"lstchain_create_calibration_file",
f"--input_file={input_file}",
f"--output_file={output_file}",
"--LSTEventSource.default_trigger_type=tib",
f"--EventSource.min_flatfield_adc={min_ff}",
f"--EventSource.max_flatfield_adc={max_ff}",
f"--LSTCalibrationCalculator.systematic_correction_path={systematics_file}",
f"--LSTEventSource.EventTimeCalculator.run_summary_path={run_summary_path}",
f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={time_file}",
f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={pedestal_file}",
f"--LSTEventSource.use_flatfield_heuristic={args.use_flatfield_heuristic}",
f"--FlatFieldCalculator.sample_size={stat_events}",
f"--PedestalCalculator.sample_size={stat_events}",
f"--config={config_file}",
f"--log-file={log_file}",
"--log-file-level=DEBUG",
]
print("\n--> RUNNING...")
subprocess.run(cmd, check=True)
# plot and save some results
plot_file = f"{output_dir}/log/{output_name}.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
calib.read_file(output_file, tel_id)
calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data, run, plot_file)
print("\n--> END")
def main():
args = parser.parse_args()
date = args.date
prod_id = args.prod_version
base_dir = args.base_dir
sub_run = args.sub_run
config_file = args.config
prefix = args.input_prefix
yes = args.yes
pro_symlink = not args.no_pro_symlink
calib_dir = base_dir / LEVEL_A_PIXEL_DIR
# verify config file
if not config_file.exists():
raise IOError(f"Config file {config_file} does not exists.")
print(f"\n--> Config file {config_file}")
# verify output dir
output_dir = calib_dir / "ffactor_systematics" / date / prod_id
if not output_dir.exists():
print(f"--> Create directory {output_dir}")
output_dir.mkdir(parents=True, exist_ok=True)
if pro_symlink:
pro = "pro"
create_pro_symlink(output_dir)
else:
pro = prod_id
# verify input dir
input_dir = calib_dir / "calibration" / date / pro
if not input_dir.exists():
raise IOError(f"Input directory {input_dir} not found")
print(f"\n--> Input directory {input_dir}")
# make log dir
log_dir = output_dir / "log"
if not log_dir.exists():
print(f"--> Create directory {log_dir}")
log_dir.mkdir(parents=True, exist_ok=True)
# define output file names
output_file = output_dir / f"{prefix}_scan_fit_{date}.{sub_run:04d}.h5"
log_file = log_dir / f"{prefix}_scan_fit_{date}.{sub_run:04d}.log"
plot_file = log_dir / f"{prefix}_scan_fit_{date}.{sub_run:04d}.pdf"
if output_file.exists():
remove = False
if not yes and os.getenv('SLURM_JOB_ID') is None:
remove = query_yes_no(">>> Output file exists already. Do you want to remove it?")
if yes or remove:
os.remove(output_file)
os.remove(log_file)
else:
print("\n--> Output file exists already. Stop")
exit(1)
print(f"\n--> Plot file {plot_file}")
print(f"\n--> Log file {log_file}")
#
# produce intensity scan fit file
#
cmd = [
"lstchain_fit_intensity_scan",
f"--config={config_file}",
f"--input_dir={input_dir}",
f"--output_path={output_file}",
f"--plot_path={plot_file}",
f"--sub_run={sub_run}",
f"--input_prefix={prefix}",
f"--log-file={log_file}",
"--log-file-level=DEBUG",
]
print("\n--> RUNNING...")
subprocess.run(cmd, check=True)
print("\n--> END")
def main():
args = parser.parse_args()
run = args.run_number
prod_id = args.prod_version
max_events = args.max_events
base_dir = args.base_dir
tel_id = args.tel_id
yes = args.yes
pro_symlink = not args.no_pro_symlink
print(f"\n--> Start calculating DRS4 pedestals from run {run}\n")
# verify input file
r0_dir = args.r0_dir or Path(args.base_dir) / 'R0'
input_file = find_r0_subrun(run, sub_run=0, r0_dir=r0_dir)
date = input_file.parent.name
# verify and make output dir
calib_dir = base_dir / LEVEL_A_PIXEL_DIR
output_dir = calib_dir / "drs4_baseline" / date / prod_id
if not output_dir.exists():
print(f"--> Create directory {output_dir}")
output_dir.mkdir(parents=True, exist_ok=True)
# update the default production directory
if pro_symlink:
create_pro_symlink(output_dir)
# make log dir
log_dir = output_dir / "log"
if not log_dir.exists():
print(f"--> Create directory {log_dir}")
os.makedirs(log_dir, exist_ok=True)
# define output file
output_file = output_dir / f"drs4_pedestal.Run{run:05d}.0000.h5"
if output_file.exists():
remove = False
if not yes and os.getenv('SLURM_JOB_ID') is None:
remove = query_yes_no(
">>> Output file exists already. Do you want to remove it?")
if yes or remove:
os.remove(output_file)
else:
print("\n--> Output file exists already. Stop")
exit(1)
# run lstchain script
cmd = [
"lstchain_create_drs4_pedestal_file",
f"--input={input_file}",
f"--output={output_file}",
f"--max-events={max_events}",
]
if args.no_progress:
cmd.append("--no-progress")
subprocess.run(cmd, check=True)
# plot and save some results
plot_file = f"{output_dir}/log/drs4_pedestal.Run{run:05d}.0000.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
drs4.plot_pedestals(input_file,
output_file,
run,
plot_file,
tel_id=tel_id,
offset_value=400)
print("\n--> END")
def main():
print(f"\n--> Start calculating calibration from run {run}")
try:
# verify input file
file_list = sorted(Path(f"{base_dir}/R0").rglob(f'*{run}.0000*'))
if len(file_list) == 0:
print(f">>> Error: Run {run} not found\n")
raise NameError()
else:
input_file = file_list[0]
# find date
input_dir, name = os.path.split(os.path.abspath(input_file))
path, date = input_dir.rsplit('/', 1)
# verify output dir
output_dir = f"{base_dir}/calibration/{date}/{prod_id}"
if not os.path.exists(output_dir):
print(f">>> Error: The output directory {output_dir} do not exist")
print(
f">>> You must create the drs4 pedestal file to create it.\n Exit"
)
exit(0)
# search the pedestal calibration file
pedestal_file = f"{output_dir}/drs4_pedestal.Run{ped_run}.0000.fits"
if not os.path.exists(pedestal_file):
print(
f">>> Error: The pedestal file {pedestal_file} do not exist.\n Exit"
)
exit(0)
# define config file
config_file = os.path.join(
os.path.dirname(__file__),
"../../data/onsite_camera_calibration_param.json")
if not os.path.exists(config_file):
print(f">>> Config file {config_file} do not exists. \n Exit ")
exit(1)
print(f"\n--> Config file {config_file}")
#
# produce drs4 time calibration file
#
time_file = f"{output_dir}/time_calibration.Run{run}.0000.hdf5"
print(f"\n***** PRODUCE TIME CALIBRATION FILE ***** ")
if default_time_run is 0:
print(f"\n--> PRODUCING TIME CALIBRATION in {time_file} ...")
cmd = f"lstchain_data_create_time_calibration_file --input-file {input_file} " \
f"--output-file {time_file} --config {config_file} --pedestal-file {pedestal_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
else:
# otherwise perform a link to the default time calibration file
print(
f"\n--> PRODUCING LINK TO DEFAULT TIME CALIBRATION (run {default_time_run})"
)
file_list = sorted(
Path(f"{base_dir}/calibration/").rglob(
f'*/{prod_id}/time_calibration.Run{default_time_run}*'))
if len(file_list) == 0:
print(
f">>> Error: time calibration file for run {default_time_run} not found\n"
)
raise NameError()
else:
time_calibration_file = file_list[0]
input_dir, name = os.path.split(
os.path.abspath(time_calibration_file))
cmd = f"ln -sf {time_calibration_file} {time_file}"
os.system(cmd)
print(f"\n--> Time calibration file: {time_file}")
# define charge file names
print(f"\n***** PRODUCE CHARGE CALIBRATION FILE ***** ")
output_file = f"{output_dir}/calibration.Run{run}.0000.hdf5"
log_file = f"{output_dir}/log/calibration.Run{run}.0000.log"
print(f"\n--> Output file {output_file}")
if os.path.exists(output_file) and ff_calibration == 'yes':
if query_yes_no(
">>> Output file exists already. Do you want to remove it?"
):
os.remove(output_file)
else:
print(f"\n--> Stop")
exit(1)
print(f"\n--> Log file {log_file}")
#
# produce ff calibration file
#
if ff_calibration == 'yes':
# run lstchain script
cmd = f"lstchain_create_calibration_file " \
f"--input_file={input_file} --output_file={output_file} --pedestal_file={pedestal_file} " \
f"--FlasherFlatFieldCalculator.time_calibration_path={time_file} --FlasherFlatFieldCalculator.sample_size={stat_events} "\
f"--PedestalIntegrator.sample_size={stat_events} " \
f"--EventSource.max_events={max_events} --config={config_file} > {log_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
# plot and save some results
plot_file = f"{output_dir}/log/calibration.Run{run}.0000.pedestal.Run{ped_run}.0000.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
calib.read_file(output_file, tel_id)
calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data,
run, plot_file)
print("\n--> END")
except Exception as e:
print(f"\n >>> Exception: {e}")