def test_plot_calib(temp_dir_observed_files): pdf_filename = temp_dir_observed_files / "calibration.Run2006.0000.pdf" calib.read_file(test_calib_path) calib.plot_all( calib.ped_data, calib.ff_data, calib.calib_data, run=2006, plot_file=pdf_filename, ) assert pdf_filename.is_file()
def main(): print(f"\n--> Start calculating calibration from run {run}") try: # verify input file file_list = sorted( Path(f"{base_dir}/R0").rglob(f'*{run}.{sub_run:04d}*')) if len(file_list) == 0: raise IOError(f"Run {run} not found\n") else: input_file = file_list[0] print(f"\n--> Input file: {input_file}") # find date input_dir, name = os.path.split(os.path.abspath(input_file)) path, date = input_dir.rsplit('/', 1) # verify output dir output_dir = f"{base_dir}/calibration/{date}/{prod_id}" if not os.path.exists(output_dir): raise IOError(f"Output directory {output_dir} does not exist\n") # search the pedestal calibration file pedestal_file = f"{output_dir}/drs4_pedestal.Run{ped_run:05d}.0000.fits" if not os.path.exists(pedestal_file): raise IOError(f"Pedestal file {pedestal_file} does not exist.\n ") # search the summary file info run_summary_path = f"{base_dir}/monitoring/RunSummary/RunSummary_{date}.ecsv" if not os.path.exists(run_summary_path): raise IOError( f"Night summary file {run_summary_path} does not exist\n") # define config file config_file = os.path.join( os.path.dirname(__file__), "../../data/onsite_camera_calibration_param.json") if not os.path.exists(config_file): raise IOError(f"Config file {config_file} does not exists. \n") print(f"\n--> Config file {config_file}") # # produce drs4 time calibration file # time_file = f"{output_dir}/time_calibration.Run{run:05d}.0000.h5" print(f"\n***** PRODUCE TIME CALIBRATION FILE ***** ") if default_time_run is 0: print(f"\n--> PRODUCING TIME CALIBRATION in {time_file} ...") cmd = f"lstchain_data_create_time_calibration_file --input-file {input_file} " \ f"--output-file {time_file} --config {config_file} " \ f"--run-summary-file={run_summary_path} " \ f"--pedestal-file {pedestal_file} 2>&1" print("\n--> RUNNING...") os.system(cmd) else: # otherwise perform a link to the default time calibration file print( f"\n--> PRODUCING LINK TO DEFAULT TIME CALIBRATION (run {default_time_run})" ) file_list = sorted( Path(f"{base_dir}/calibration/").rglob( f'*/{prod_id}/time_calibration.Run{default_time_run}*')) if len(file_list) == 0: raise IOError( f"Time calibration file for run {default_time_run} not found\n" ) else: time_calibration_file = file_list[0] cmd = f"ln -sf {time_calibration_file} {time_file}" os.system(cmd) print(f"\n--> Time calibration file: {time_file}") # define charge file names print(f"\n***** PRODUCE CHARGE CALIBRATION FILE ***** ") output_file = f"{output_dir}/calibration.Run{run:05d}.{sub_run:04d}.h5" log_file = f"{output_dir}/log/calibration.Run{run:05d}.{sub_run:04d}.log" print(f"\n--> Output file {output_file}") if os.path.exists(output_file) and ff_calibration == 'yes': if query_yes_no( ">>> Output file exists already. Do you want to remove it?" ): os.remove(output_file) else: print(f"\n--> Stop") exit(1) print(f"\n--> Log file {log_file}") # # produce ff calibration file # if ff_calibration == 'yes': # run lstchain script cmd = f"lstchain_create_calibration_file " \ f"--input_file={input_file} --output_file={output_file} "\ f"--EventSource.max_events={max_events} " \ f"--EventSource.default_trigger_type=tib " \ f"--EventSource.min_flatfield_adc={min_ff} " \ f"--EventSource.max_flatfield_adc={max_ff} " \ f"--LSTEventSource.EventTimeCalculator.run_summary_path={run_summary_path} " \ f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={time_file} " \ f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={pedestal_file} " \ f"--FlatFieldCalculator.sample_size={stat_events} --PedestalCalculator.sample_size={stat_events} " \ f"--config={config_file} > {log_file} 2>&1" print("\n--> RUNNING...") os.system(cmd) # plot and save some results plot_file = f"{output_dir}/log/calibration.Run{run:05d}.{sub_run:04d}.pedestal.Run{ped_run:05d}.0000.pdf" print(f"\n--> PRODUCING PLOTS in {plot_file} ...") calib.read_file(output_file, tel_id) calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data, run, plot_file) print("\n--> END") except Exception as e: print(f"\n >>> Exception: {e}")
def main(): args = parser.parse_args() run = args.run_number prod_id = args.prod_version stat_events = args.statistics time_run = args.time_run sys_date = args.sys_date no_sys_correction = args.no_sys_correction output_base_name = args.output_base_name sub_run = args.sub_run tel_id = args.tel_id config_file = args.config yes = args.yes pro_symlink = not args.no_pro_symlink # looks for the filter values in the database if not given if args.filters is None: filters = search_filter(run, args.mongodb) else: filters = args.filters if filters is None: sys.exit(f"Missing filter value for run {run}. \n") # define the FF selection cuts if args.min_ff is None or args.max_ff is None: min_ff, max_ff = define_FF_selection_range(filters) else: min_ff, max_ff = args.min_ff, args.max_ff print(f"\n--> Start calculating calibration from run {run}, filters {filters}") # verify config file if not config_file.exists(): raise IOError(f"Config file {config_file} does not exists. \n") print(f"\n--> Config file {config_file}") # verify input file r0_dir = args.r0_dir or args.base_dir / 'R0' input_file = find_r0_subrun(run, sub_run, r0_dir) date = input_file.parent.name print(f"\n--> Input file: {input_file}") # verify output dir calib_dir = args.base_dir / LEVEL_A_PIXEL_DIR output_dir = calib_dir / "calibration" / date / prod_id if not output_dir.exists(): print(f"--> Create directory {output_dir}") output_dir.mkdir(parents=True, exist_ok=True) if pro_symlink: pro = "pro" create_pro_symlink(output_dir) else: pro = prod_id # make log dir log_dir = output_dir / "log" if not log_dir.exists(): print(f"--> Create directory {log_dir}") log_dir.mkdir(parents=True, exist_ok=True) # search the summary file info run_summary_path = find_run_summary(date, args.base_dir) print(f"\n--> Use run summary {run_summary_path}") pedestal_file = find_pedestal_file(pro, args.pedestal_run, date=date, base_dir=args.base_dir) print(f"\n--> Pedestal file: {pedestal_file}") # search for time calibration file time_file = find_time_calibration_file(pro, run, time_run, args.base_dir) print(f"\n--> Time calibration file: {time_file}") # define systematic correction file if no_sys_correction: systematics_file = None else: systematics_file = find_systematics_correction_file(pro, date, sys_date, args.base_dir) print(f"\n--> F-factor systematics correction file: {systematics_file}") # define charge file names print("\n***** PRODUCE CHARGE CALIBRATION FILE ***** ") if filters is not None: filter_info = f"_filters_{filters}" else: filter_info = "" # remember there are no systematic corrections prefix = "no_sys_corrected_" if no_sys_correction else "" output_name = f"{prefix}{output_base_name}{filter_info}.Run{run:05d}.{sub_run:04d}" output_file = output_dir / f'{output_name}.h5' print(f"\n--> Output file {output_file}") log_file = log_dir / f"{output_name}.log" print(f"\n--> Log file {log_file}") if output_file.exists(): remove = False if not yes and os.getenv('SLURM_JOB_ID') is None: remove = query_yes_no(">>> Output file exists already. Do you want to remove it?") if yes or remove: os.remove(output_file) os.remove(log_file) else: print("\n--> Output file exists already. Stop") exit(1) # # produce ff calibration file # cmd = [ "lstchain_create_calibration_file", f"--input_file={input_file}", f"--output_file={output_file}", "--LSTEventSource.default_trigger_type=tib", f"--EventSource.min_flatfield_adc={min_ff}", f"--EventSource.max_flatfield_adc={max_ff}", f"--LSTCalibrationCalculator.systematic_correction_path={systematics_file}", f"--LSTEventSource.EventTimeCalculator.run_summary_path={run_summary_path}", f"--LSTEventSource.LSTR0Corrections.drs4_time_calibration_path={time_file}", f"--LSTEventSource.LSTR0Corrections.drs4_pedestal_path={pedestal_file}", f"--LSTEventSource.use_flatfield_heuristic={args.use_flatfield_heuristic}", f"--FlatFieldCalculator.sample_size={stat_events}", f"--PedestalCalculator.sample_size={stat_events}", f"--config={config_file}", f"--log-file={log_file}", "--log-file-level=DEBUG", ] print("\n--> RUNNING...") subprocess.run(cmd, check=True) # plot and save some results plot_file = f"{output_dir}/log/{output_name}.pdf" print(f"\n--> PRODUCING PLOTS in {plot_file} ...") calib.read_file(output_file, tel_id) calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data, run, plot_file) print("\n--> END")
def main(): print("input files: {}".format(args.input_file)) print("calib file: {}".format(args.calib_file)) print("output file: {}".format(args.output_file)) max_muons = args.max_muons # Definition of the output parameters for the table output_parameters = create_muon_table() if args.calib_file is not None: plot_calib.read_file(args.calib_file) bad_pixels = plot_calib.calib_data.unusable_pixels[0] print(f"Found a total of {np.sum(bad_pixels)} bad pixels.") # image = pd.read_hdf(args.input_file, key = dl1_image_lstcam_key) # The call above does not work, because of the file's vector columns (pixel-wise charges & times) # So we use tables for the time being. print(glob.glob(args.input_file)) filenames = glob.glob(args.input_file) filenames.sort() lst1_tel_id = 1 num_muons = 0 for filename in filenames: print('Opening file', filename) cam_description_table = Table.read( filename, path="instrument/telescope/camera/LSTCam") geom = CameraGeometry.from_table(cam_description_table) subarray = read_subarray_description(filename, subarray_name='LST-1') images = Table.read(filename, path=dl1_images_lstcam_key)['image'] parameters = pd.read_hdf(filename, key=dl1_params_lstcam_key) telescope_description = read_telescopes_descriptions( filename)[lst1_tel_id] equivalent_focal_length = telescope_description.optics.equivalent_focal_length mirror_area = telescope_description.optics.mirror_area # fill dummy event times with NaNs in case they do not exist (like in MC): if 'dragon_time' not in parameters.keys(): dummy_times = np.empty(len(parameters['event_id'])) dummy_times[:] = np.nan parameters['dragon_time'] = dummy_times for full_image, event_id, dragon_time in zip( images, parameters['event_id'], parameters['dragon_time']): if args.calib_file is not None: image = full_image * (~bad_pixels) else: image = full_image # print("Event {}. Number of pixels above 10 phe: {}".format(event_id, # np.size(image[image > 10.]))) # if((np.size(image[image > 10.]) > 300) or (np.size(image[image > 10.]) < 50)): # continue if not tag_pix_thr( image): # default skips pedestal and calibration events continue # default values apply no filtering. # This filter is rather useless for biased extractors anyway # if not muon_filter(image) # continue (muonintensityparam, dist_mask, size, size_outside_ring, muonringparam, good_ring, radial_distribution, mean_pixel_charge_around_ring, muonparameters) = analyze_muon_event(subarray, event_id, image, geom, equivalent_focal_length, mirror_area, args.plot_rings, args.plots_path) if good_ring: num_muons += 1 print("Number of good muon rings found {}, EventID {}".format( num_muons, event_id)) # write ring data, including also "not-so-good" rings # in case we want to reconsider ring selections!: fill_muon_event(parameters, output_parameters, good_ring, event_id, dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, size, size_outside_ring, mean_pixel_charge_around_ring, muonparameters) if max_muons is not None and num_muons == max_muons: break if max_muons is not None and num_muons == max_muons: break table = Table(output_parameters) table.write(args.output_file, format='fits', overwrite=True)
def main(): print(f"\n--> Start calculating calibration from run {run}") try: # verify input file file_list = sorted(Path(f"{base_dir}/R0").rglob(f'*{run}.0000*')) if len(file_list) == 0: print(f">>> Error: Run {run} not found\n") raise NameError() else: input_file = file_list[0] # find date input_dir, name = os.path.split(os.path.abspath(input_file)) path, date = input_dir.rsplit('/', 1) # verify output dir output_dir = f"{base_dir}/calibration/{date}/{prod_id}" if not os.path.exists(output_dir): print(f">>> Error: The output directory {output_dir} do not exist") print( f">>> You must create the drs4 pedestal file to create it.\n Exit" ) exit(0) # search the pedestal calibration file pedestal_file = f"{output_dir}/drs4_pedestal.Run{ped_run}.0000.fits" if not os.path.exists(pedestal_file): print( f">>> Error: The pedestal file {pedestal_file} do not exist.\n Exit" ) exit(0) # # produce ff calibration file # # define charge file names output_file = f"{output_dir}/calibration.Run{run}.0000.hdf5" log_file = f"{output_dir}/log/calibration.Run{run}.0000.log" print(f"\n--> Output file {output_file}") if os.path.exists(output_file) and ff_calibration is 'yes': if query_yes_no( ">>> Output file exists already. Do you want to remove it?" ): os.remove(output_file) else: print(f"\n--> Stop") exit(1) print(f"\n--> Log file {log_file}") # define config file config_file = os.path.join( os.path.dirname(__file__), "../../data/onsite_camera_calibration_param.json") if not os.path.exists(config_file): print(f">>> Config file {config_file} do not exists. \n Exit ") exit(1) print(f"\n--> Config file {config_file}") if ff_calibration is 'yes': # run lstchain script cmd = f"lstchain_data_create_calibration_file " \ f"--input_file={input_file} --output_file={output_file} --pedestal_file={pedestal_file} " \ f"--FlatFieldCalculator.sample_size={stat_events} --PedestalCalculator.sample_size={stat_events} " \ f"--EventSource.max_events={max_events} --config={config_file} > {log_file} 2>&1" print("\n--> RUNNING...") os.system(cmd) # plot and save some results plot_file = f"{output_dir}/log/calibration.Run{run}.0000.pedestal.Run{ped_run}.0000.pdf" print(f"\n--> PRODUCING PLOTS in {plot_file} ...") calib.read_file(output_file, tel_id) calib.plot_all(calib.ped_data, calib.ff_data, calib.calib_data, run, plot_file) # # produce drs4 time calibration file # time_file = f"{output_dir}/time_calibration.Run{run}.0000.hdf5" if default_time_run is 0: print(f"\n--> PRODUCING TIME CALIBRATION in {time_file} ...") cmd = f"lstchain_data_create_time_calibration_file --input_file {input_file} " \ f"--output_file {time_file} -conf {config_file} -ped {pedestal_file} 2>&1" print("\n--> RUNNING...") os.system(cmd) else: # otherwise perform a link to the default time calibration file print( f"\n--> PRODUCING LINK TO DEFAULT TIME CALIBRATION (run {default_time_run})" ) file_list = sorted( Path(f"{base_dir}/calibration/").rglob( f'*/{prod_id}/time_calibration.Run{default_time_run}*')) if len(file_list) == 0: print( f">>> Error: time calibration file for run {default_time_run} not found\n" ) raise NameError() else: time_calibration_file = file_list[0] input_dir, name = os.path.split( os.path.abspath(time_calibration_file)) cmd = f"ln -sf {time_calibration_file} {time_file}" os.system(cmd) print(f"\n--> Time calibration file: {time_file}") print("\n--> END") except Exception as e: print(f"\n >>> Exception: {e}")