def test_plot_calib(temp_dir_observed_files):
pdf_filename = temp_dir_observed_files / "calibration.Run2006.0000.pdf"
calib.read_file(test_calib_path)
calib.plot_all(
calib.ped_data,
calib.ff_data,
calib.calib_data,
run=2006,
plot_file=pdf_filename,
)
assert pdf_filename.is_file()
def main():
print(f"\n--> Start calculating calibration from run {run}")
try:
# verify input file
file_list = sorted(
Path(f"{base_dir}/R0").rglob(f'*{run}.{sub_run:04d}*'))
if len(file_list) == 0:
raise IOError(f"Run {run} not found\n")
else:
input_file = file_list[0]
print(f"\n--> Input file: {input_file}")
# find date
input_dir, name = os.path.split(os.path.abspath(input_file))
path, date = input_dir.rsplit('/', 1)
# verify output dir
output_dir = f"{base_dir}/calibration/{date}/{prod_id}"
if not os.path.exists(output_dir):
raise IOError(f"Output directory {output_dir} does not exist\n")
# search the pedestal calibration file
pedestal_file = f"{output_dir}/drs4_pedestal.Run{ped_run:05d}.0000.fits"
if not os.path.exists(pedestal_file):
raise IOError(f"Pedestal file {pedestal_file} does not exist.\n ")
# search the summary file info
run_summary_path = f"{base_dir}/monitoring/RunSummary/RunSummary_{date}.ecsv"
if not os.path.exists(run_summary_path):
raise IOError(
f"Night summary file {run_summary_path} does not exist\n")
# define config file
config_file = os.path.join(
os.path.dirname(__file__),
"../../data/onsite_camera_calibration_param.json")
if not os.path.exists(config_file):
raise IOError(f"Config file {config_file} does not exists. \n")
print(f"\n--> Config file {config_file}")
#
# produce drs4 time calibration file
#
time_file = f"{output_dir}/time_calibration.Run{run:05d}.0000.h5"
print(f"\n***** PRODUCE TIME CALIBRATION FILE ***** ")
if default_time_run is 0:
print(f"\n--> PRODUCING TIME CALIBRATION in {time_file} ...")
cmd = f"lstchain_data_create_time_calibration_file --input-file {input_file} " \
f"--output-file {time_file} --config {config_file} " \
f"--run-summary-file={run_summary_path} " \
f"--pedestal-file {pedestal_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
else:
# otherwise perform a link to the default time calibration file
print(
f"\n--> PRODUCING LINK TO DEFAULT TIME CALIBRATION (run {default_time_run})"
)
file_list = sorted(
Path(f"{base_dir}/calibration/").rglob(
f'*/{prod_id}/time_calibration.Run{default_time_run}*'))
if len(file_list) == 0:
raise IOError(
f"Time calibration file for run {default_time_run} not found\n"
)
else:
time_calibration_file = file_list[0]
cmd = f"ln -sf {time_calibration_file} {time_file}"
os.system(cmd)
print(f"\n--> Time calibration file: {time_file}")
# define charge file names
print(f"\n***** PRODUCE CHARGE CALIBRATION FILE ***** ")
output_file = f"{output_dir}/calibration.Run{run:05d}.{sub_run:04d}.h5"
log_file = f"{output_dir}/log/calibration.Run{run:05d}.{sub_run:04d}.log"
print(f"\n--> Output file {output_file}")
if os.path.exists(output_file) and ff_calibration == 'yes':
if query_yes_no(
">>> Output file exists already. Do you want to remove it?"
):
os.remove(output_file)
else:
print(f"\n--> Stop")
exit(1)
print(f"\n--> Log file {log_file}")
#
# produce ff calibration file
#
if ff_calibration == 'yes':
# run lstchain script
cmd = f"lstchain_create_calibration_file " \
f"--input_file={input_file} --output_file={output_file} "\
f"--EventSource.max_events={max_events} " \
f"--EventSource.default_trigger_type=tib " \
f"--EventSource.min_flatfield_adc={min_ff} " \
f"--EventSource.max_flatfield_adc={max_ff} " \
f"--LSTEventSource.EventTimeCalculator.run_summary_path={run_summary_path} " \
f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={time_file} " \
f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={pedestal_file} " \
f"--FlatFieldCalculator.sample_size={stat_events} --PedestalCalculator.sample_size={stat_events} " \
f"--config={config_file} > {log_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
# plot and save some results
plot_file = f"{output_dir}/log/calibration.Run{run:05d}.{sub_run:04d}.pedestal.Run{ped_run:05d}.0000.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
calib.read_file(output_file, tel_id)
calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data,
run, plot_file)
print("\n--> END")
except Exception as e:
print(f"\n >>> Exception: {e}")
def main():
args = parser.parse_args()
run = args.run_number
prod_id = args.prod_version
stat_events = args.statistics
time_run = args.time_run
sys_date = args.sys_date
no_sys_correction = args.no_sys_correction
output_base_name = args.output_base_name
sub_run = args.sub_run
tel_id = args.tel_id
config_file = args.config
yes = args.yes
pro_symlink = not args.no_pro_symlink
# looks for the filter values in the database if not given
if args.filters is None:
filters = search_filter(run, args.mongodb)
else:
filters = args.filters
if filters is None:
sys.exit(f"Missing filter value for run {run}. \n")
# define the FF selection cuts
if args.min_ff is None or args.max_ff is None:
min_ff, max_ff = define_FF_selection_range(filters)
else:
min_ff, max_ff = args.min_ff, args.max_ff
print(f"\n--> Start calculating calibration from run {run}, filters {filters}")
# verify config file
if not config_file.exists():
raise IOError(f"Config file {config_file} does not exists. \n")
print(f"\n--> Config file {config_file}")
# verify input file
r0_dir = args.r0_dir or args.base_dir / 'R0'
input_file = find_r0_subrun(run, sub_run, r0_dir)
date = input_file.parent.name
print(f"\n--> Input file: {input_file}")
# verify output dir
calib_dir = args.base_dir / LEVEL_A_PIXEL_DIR
output_dir = calib_dir / "calibration" / date / prod_id
if not output_dir.exists():
print(f"--> Create directory {output_dir}")
output_dir.mkdir(parents=True, exist_ok=True)
if pro_symlink:
pro = "pro"
create_pro_symlink(output_dir)
else:
pro = prod_id
# make log dir
log_dir = output_dir / "log"
if not log_dir.exists():
print(f"--> Create directory {log_dir}")
log_dir.mkdir(parents=True, exist_ok=True)
# search the summary file info
run_summary_path = find_run_summary(date, args.base_dir)
print(f"\n--> Use run summary {run_summary_path}")
pedestal_file = find_pedestal_file(pro, args.pedestal_run, date=date, base_dir=args.base_dir)
print(f"\n--> Pedestal file: {pedestal_file}")
# search for time calibration file
time_file = find_time_calibration_file(pro, run, time_run, args.base_dir)
print(f"\n--> Time calibration file: {time_file}")
# define systematic correction file
if no_sys_correction:
systematics_file = None
else:
systematics_file = find_systematics_correction_file(pro, date, sys_date, args.base_dir)
print(f"\n--> F-factor systematics correction file: {systematics_file}")
# define charge file names
print("\n***** PRODUCE CHARGE CALIBRATION FILE ***** ")
if filters is not None:
filter_info = f"_filters_{filters}"
else:
filter_info = ""
# remember there are no systematic corrections
prefix = "no_sys_corrected_" if no_sys_correction else ""
output_name = f"{prefix}{output_base_name}{filter_info}.Run{run:05d}.{sub_run:04d}"
output_file = output_dir / f'{output_name}.h5'
print(f"\n--> Output file {output_file}")
log_file = log_dir / f"{output_name}.log"
print(f"\n--> Log file {log_file}")
if output_file.exists():
remove = False
if not yes and os.getenv('SLURM_JOB_ID') is None:
remove = query_yes_no(">>> Output file exists already. Do you want to remove it?")
if yes or remove:
os.remove(output_file)
os.remove(log_file)
else:
print("\n--> Output file exists already. Stop")
exit(1)
#
# produce ff calibration file
#
cmd = [
"lstchain_create_calibration_file",
f"--input_file={input_file}",
f"--output_file={output_file}",
"--LSTEventSource.default_trigger_type=tib",
f"--EventSource.min_flatfield_adc={min_ff}",
f"--EventSource.max_flatfield_adc={max_ff}",
f"--LSTCalibrationCalculator.systematic_correction_path={systematics_file}",
f"--LSTEventSource.EventTimeCalculator.run_summary_path={run_summary_path}",
f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={time_file}",
f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={pedestal_file}",
f"--LSTEventSource.use_flatfield_heuristic={args.use_flatfield_heuristic}",
f"--FlatFieldCalculator.sample_size={stat_events}",
f"--PedestalCalculator.sample_size={stat_events}",
f"--config={config_file}",
f"--log-file={log_file}",
"--log-file-level=DEBUG",
]
print("\n--> RUNNING...")
subprocess.run(cmd, check=True)
# plot and save some results
plot_file = f"{output_dir}/log/{output_name}.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
calib.read_file(output_file, tel_id)
calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data, run, plot_file)
print("\n--> END")
def main():
print("input files: {}".format(args.input_file))
print("calib file: {}".format(args.calib_file))
print("output file: {}".format(args.output_file))
max_muons = args.max_muons
# Definition of the output parameters for the table
output_parameters = create_muon_table()
if args.calib_file is not None:
plot_calib.read_file(args.calib_file)
bad_pixels = plot_calib.calib_data.unusable_pixels[0]
print(f"Found a total of {np.sum(bad_pixels)} bad pixels.")
# image = pd.read_hdf(args.input_file, key = dl1_image_lstcam_key)
# The call above does not work, because of the file's vector columns (pixel-wise charges & times)
# So we use tables for the time being.
print(glob.glob(args.input_file))
filenames = glob.glob(args.input_file)
filenames.sort()
lst1_tel_id = 1
num_muons = 0
for filename in filenames:
print('Opening file', filename)
cam_description_table = Table.read(
filename, path="instrument/telescope/camera/LSTCam")
geom = CameraGeometry.from_table(cam_description_table)
subarray = read_subarray_description(filename, subarray_name='LST-1')
images = Table.read(filename, path=dl1_images_lstcam_key)['image']
parameters = pd.read_hdf(filename, key=dl1_params_lstcam_key)
telescope_description = read_telescopes_descriptions(
filename)[lst1_tel_id]
equivalent_focal_length = telescope_description.optics.equivalent_focal_length
mirror_area = telescope_description.optics.mirror_area
# fill dummy event times with NaNs in case they do not exist (like in MC):
if 'dragon_time' not in parameters.keys():
dummy_times = np.empty(len(parameters['event_id']))
dummy_times[:] = np.nan
parameters['dragon_time'] = dummy_times
for full_image, event_id, dragon_time in zip(
images, parameters['event_id'], parameters['dragon_time']):
if args.calib_file is not None:
image = full_image * (~bad_pixels)
else:
image = full_image
# print("Event {}. Number of pixels above 10 phe: {}".format(event_id,
# np.size(image[image > 10.])))
# if((np.size(image[image > 10.]) > 300) or (np.size(image[image > 10.]) < 50)):
# continue
if not tag_pix_thr(
image): # default skips pedestal and calibration events
continue
# default values apply no filtering.
# This filter is rather useless for biased extractors anyway
# if not muon_filter(image)
# continue
(muonintensityparam, dist_mask, size, size_outside_ring,
muonringparam, good_ring, radial_distribution,
mean_pixel_charge_around_ring,
muonparameters) = analyze_muon_event(subarray, event_id, image,
geom,
equivalent_focal_length,
mirror_area, args.plot_rings,
args.plots_path)
if good_ring:
num_muons += 1
print("Number of good muon rings found {}, EventID {}".format(
num_muons, event_id))
# write ring data, including also "not-so-good" rings
# in case we want to reconsider ring selections!:
fill_muon_event(parameters, output_parameters, good_ring, event_id,
dragon_time, muonintensityparam, dist_mask,
muonringparam, radial_distribution, size,
size_outside_ring, mean_pixel_charge_around_ring,
muonparameters)
if max_muons is not None and num_muons == max_muons:
break
if max_muons is not None and num_muons == max_muons:
break
table = Table(output_parameters)
table.write(args.output_file, format='fits', overwrite=True)
def main():
print(f"\n--> Start calculating calibration from run {run}")
try:
# verify input file
file_list = sorted(Path(f"{base_dir}/R0").rglob(f'*{run}.0000*'))
if len(file_list) == 0:
print(f">>> Error: Run {run} not found\n")
raise NameError()
else:
input_file = file_list[0]
# find date
input_dir, name = os.path.split(os.path.abspath(input_file))
path, date = input_dir.rsplit('/', 1)
# verify output dir
output_dir = f"{base_dir}/calibration/{date}/{prod_id}"
if not os.path.exists(output_dir):
print(f">>> Error: The output directory {output_dir} do not exist")
print(
f">>> You must create the drs4 pedestal file to create it.\n Exit"
)
exit(0)
# search the pedestal calibration file
pedestal_file = f"{output_dir}/drs4_pedestal.Run{ped_run}.0000.fits"
if not os.path.exists(pedestal_file):
print(
f">>> Error: The pedestal file {pedestal_file} do not exist.\n Exit"
)
exit(0)
#
# produce ff calibration file
#
# define charge file names
output_file = f"{output_dir}/calibration.Run{run}.0000.hdf5"
log_file = f"{output_dir}/log/calibration.Run{run}.0000.log"
print(f"\n--> Output file {output_file}")
if os.path.exists(output_file) and ff_calibration is 'yes':
if query_yes_no(
">>> Output file exists already. Do you want to remove it?"
):
os.remove(output_file)
else:
print(f"\n--> Stop")
exit(1)
print(f"\n--> Log file {log_file}")
# define config file
config_file = os.path.join(
os.path.dirname(__file__),
"../../data/onsite_camera_calibration_param.json")
if not os.path.exists(config_file):
print(f">>> Config file {config_file} do not exists. \n Exit ")
exit(1)
print(f"\n--> Config file {config_file}")
if ff_calibration is 'yes':
# run lstchain script
cmd = f"lstchain_data_create_calibration_file " \
f"--input_file={input_file} --output_file={output_file} --pedestal_file={pedestal_file} " \
f"--FlatFieldCalculator.sample_size={stat_events} --PedestalCalculator.sample_size={stat_events} " \
f"--EventSource.max_events={max_events} --config={config_file} > {log_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
# plot and save some results
plot_file = f"{output_dir}/log/calibration.Run{run}.0000.pedestal.Run{ped_run}.0000.pdf"
print(f"\n--> PRODUCING PLOTS in {plot_file} ...")
calib.read_file(output_file, tel_id)
calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data,
run, plot_file)
#
# produce drs4 time calibration file
#
time_file = f"{output_dir}/time_calibration.Run{run}.0000.hdf5"
if default_time_run is 0:
print(f"\n--> PRODUCING TIME CALIBRATION in {time_file} ...")
cmd = f"lstchain_data_create_time_calibration_file --input_file {input_file} " \
f"--output_file {time_file} -conf {config_file} -ped {pedestal_file} 2>&1"
print("\n--> RUNNING...")
os.system(cmd)
else:
# otherwise perform a link to the default time calibration file
print(
f"\n--> PRODUCING LINK TO DEFAULT TIME CALIBRATION (run {default_time_run})"
)
file_list = sorted(
Path(f"{base_dir}/calibration/").rglob(
f'*/{prod_id}/time_calibration.Run{default_time_run}*'))
if len(file_list) == 0:
print(
f">>> Error: time calibration file for run {default_time_run} not found\n"
)
raise NameError()
else:
time_calibration_file = file_list[0]
input_dir, name = os.path.split(
os.path.abspath(time_calibration_file))
cmd = f"ln -sf {time_calibration_file} {time_file}"
os.system(cmd)
print(f"\n--> Time calibration file: {time_file}")
print("\n--> END")
except Exception as e:
print(f"\n >>> Exception: {e}")