def extract_images(input_file_or_dir_path_list,
cam_id,
output_directory=None,
max_num_img=None,
tel_id=None,
event_id=None,
rejection_criteria=None,
noise_distribution=None,
export_time_slices=False):
integrator = 'LocalPeakIntegrator'
integration_correction = False
if cam_id == "ASTRICam":
integrator_window_width = 1
integrator_window_shift = 1
elif cam_id == "CHEC":
integrator_window_width = 10
integrator_window_shift = 5
elif cam_id == "DigiCam":
integrator_window_width = 5
integrator_window_shift = 2
elif cam_id == "FlashCam":
integrator_window_width = 6
integrator_window_shift = 3
elif cam_id == "NectarCam":
integrator_window_width = 5
integrator_window_shift = 2
elif cam_id == "LSTCam":
integrator_window_width = 5
integrator_window_shift = 2
else:
raise ValueError('Unknown cam_id "{}"'.format(cam_id))
for image in image_generator(
input_file_or_dir_path_list,
max_num_images=max_num_img,
tel_filter_list=tel_id,
ev_filter_list=event_id,
cam_filter_list=[cam_id],
mc_rejection_criteria=rejection_criteria,
ctapipe_format=False,
integrator=integrator,
integrator_window_width=integrator_window_width,
integrator_window_shift=integrator_window_shift,
integration_correction=integration_correction,
mix_channels=True,
time_samples=export_time_slices):
simtel_basename = os.path.basename(image.meta['file_path'])
# INJECT NOISE IN NAN ##################################
# See https://stackoverflow.com/questions/29365194/replacing-missing-values-with-random-in-a-numpy-array
if noise_distribution is not None:
nan_mask = fill_nan_pixels(image.input_image, noise_distribution)
if noise_distribution is not None:
if image.input_samples is not None:
for sample_index in range(len(image.input_samples)):
# TODO: this is not the same noise distribution for timeslices!!!
nan_mask = fill_nan_pixels(
image.input_samples[sample_index], noise_distribution)
# SAVE THE IMAGE ##########################################
output_file_path_template = "{}_TEL{:03d}_EV{:05d}.fits"
if output_directory is not None:
prefix = os.path.join(output_directory, simtel_basename)
else:
prefix = simtel_basename
output_file_path = output_file_path_template.format(
prefix, image.meta["tel_id"], image.meta["event_id"])
print("saving", output_file_path)
save_benchmark_images(img=image.input_image,
pe_img=image.reference_image,
metadata=image.meta,
output_file_path=output_file_path,
sample_imgs=image.input_samples)
def extract_images(simtel_file_path,
tel_id_filter_list=None,
event_id_filter_list=None,
output_directory=None):
# EXTRACT IMAGES ##########################################################
# hessio_event_source returns a Python generator that streams data from an
# EventIO/HESSIO MC data file (e.g. a standard CTA data file).
# This generator contains ctapipe.core.Container instances ("event").
#
# Parameters:
# - max_events: maximum number of events to read
# - allowed_tels: select only a subset of telescope, if None, all are read.
source = hessio_event_source(simtel_file_path,
allowed_tels=tel_id_filter_list)
# ITERATE OVER EVENTS #####################################################
calib = CameraCalibrator(None, None)
for event in source:
calib.calibrate(event) # calibrate the event
event_id = int(event.dl0.event_id)
if (event_id_filter_list is None) or (event_id
in event_id_filter_list):
#print("event", event_id)
# ITERATE OVER IMAGES #############################################
for tel_id in event.trig.tels_with_trigger:
tel_id = int(tel_id)
if tel_id in tel_id_filter_list:
#print("telescope", tel_id)
# CHECK THE IMAGE GEOMETRY ################################
#print("checking geometry")
x, y = event.inst.subarray.tel[
tel_id].camera.pix_x, event.inst.subarray.tel[
tel_id].camera.pix_y
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
geom = CameraGeometry.guess(x, y, foclen)
if (geom.pix_type != "hexagonal") or (geom.cam_id !=
"DigiCam"):
raise ValueError(
"Telescope {}: error (the input image is not a valide DigiCam telescope image) -> {} ({})"
.format(tel_id, geom.pix_type, geom.cam_id))
# GET IMAGES ##############################################
pe_image = event.mc.tel[
tel_id].photo_electron_image # 1D np array
#uncalibrated_image = event.dl0.tel[tel_id].adc_sums # ctapipe 0.3.0
uncalibrated_image = event.r0.tel[
tel_id].adc_sums # ctapipe 0.4.0
pedestal = event.mc.tel[tel_id].pedestal
gain = event.mc.tel[tel_id].dc_to_pe
pixel_pos = (event.inst.subarray.tel[tel_id].camera.pix_x,
event.inst.subarray.tel[tel_id].camera.pix_y)
calibrated_image = event.dl1.tel[tel_id].image
#print(pe_image.shape)
#print(calibrated_image.shape)
#print(uncalibrated_image.shape)
#print(pedestal.shape)
#print(gain.shape)
#print(pixel_pos.shape)
#print(pixel_pos[0])
#print(pixel_pos[1])
# CONVERTING GEOMETRY (1D TO 2D) ##########################
buffer_id_str = geom.cam_id + "0"
geom2d, pe_image_2d = ctapipe_geom_converter.convert_geometry_1d_to_2d(
geom, pe_image, buffer_id_str, add_rot=0)
geom2d, calibrated_image_2d = ctapipe_geom_converter.convert_geometry_1d_to_2d(
geom, calibrated_image[0], buffer_id_str, add_rot=0)
geom2d, uncalibrated_image_2d = ctapipe_geom_converter.convert_geometry_1d_to_2d(
geom, uncalibrated_image[0], buffer_id_str, add_rot=0)
geom2d, pedestal_2d = ctapipe_geom_converter.convert_geometry_1d_to_2d(
geom, pedestal[0], buffer_id_str, add_rot=0)
geom2d, gains_2d = ctapipe_geom_converter.convert_geometry_1d_to_2d(
geom, gain[0], buffer_id_str, add_rot=0)
# Make a mock pixel position array...
pixel_pos_2d = np.array(
np.meshgrid(
np.linspace(pixel_pos[0].min(), pixel_pos[0].max(),
pe_image_2d.shape[0]),
np.linspace(pixel_pos[1].min(), pixel_pos[1].max(),
pe_image_2d.shape[1])))
# PUT NAN IN BLANK PIXELS #################################
calibrated_image_2d[np.logical_not(geom2d.mask)] = np.nan
pe_image_2d[np.logical_not(geom2d.mask)] = np.nan
uncalibrated_image_2d[np.logical_not(geom2d.mask)] = np.nan
pedestal_2d[np.logical_not(geom2d.mask)] = np.nan
gains_2d[np.logical_not(geom2d.mask)] = np.nan
pixel_pos_2d[0, np.logical_not(geom2d.mask)] = np.nan
pixel_pos_2d[1, np.logical_not(geom2d.mask)] = np.nan
###########################################################
# The ctapipe geometry converter operate on one channel
# only and then takes and return a 2D array but pywicta
# fits files keep all channels and thus takes 3D arrays...
uncalibrated_image_2d = np.array([uncalibrated_image_2d])
pedestal_2d = np.array([pedestal_2d])
gains_2d = np.array([gains_2d])
###########################################################
#print(pe_image_2d.shape)
#print(calibrated_image_2d.shape)
#print(uncalibrated_image_2d.shape)
#print(pedestal_2d.shape)
#print(gains_2d.shape)
#img = pixel_pos_2d
#print(img[1])
#import matplotlib.pyplot as plt
#im = plt.imshow(img[1])
#plt.colorbar(im)
#plt.show()
#sys.exit(0)
# GET PIXEL MASK ##########################################
pixel_mask = geom2d.mask.astype(
int
) # 1 for pixels with actual data, 0 for virtual (blank) pixels
# MAKE METADATA ###########################################
metadata = {}
metadata[
'version'] = 1 # Version of the pywicta fits format
metadata['cam_id'] = "DigiCam"
metadata['tel_id'] = tel_id
metadata['event_id'] = event_id
metadata['simtel'] = simtel_file_path
metadata['tel_trig'] = len(event.trig.tels_with_trigger)
metadata['energy'] = quantity_to_tuple(
event.mc.energy, 'TeV')
metadata['mc_az'] = quantity_to_tuple(event.mc.az, 'rad')
metadata['mc_alt'] = quantity_to_tuple(event.mc.alt, 'rad')
metadata['mc_corex'] = quantity_to_tuple(
event.mc.core_x, 'm')
metadata['mc_corey'] = quantity_to_tuple(
event.mc.core_y, 'm')
metadata['mc_hfi'] = quantity_to_tuple(
event.mc.h_first_int, 'm')
metadata['count'] = int(event.count)
metadata['run_id'] = int(event.dl0.obs_id)
metadata['tel_data'] = len(event.dl0.tels_with_data)
metadata['foclen'] = quantity_to_tuple(
event.inst.subarray.tel[tel_id].optics.
equivalent_focal_length, 'm')
metadata['tel_posx'] = quantity_to_tuple(
event.inst.subarray.tel_coords[tel_id].x, 'm')
metadata['tel_posy'] = quantity_to_tuple(
event.inst.subarray.tel_coords[tel_id].y, 'm')
metadata['tel_posz'] = quantity_to_tuple(
event.inst.subarray.tel_coords[tel_id].z, 'm')
# TODO: Astropy fails to store the following data in FITS files
#metadata['uid'] = os.getuid()
#metadata['datetime'] = str(datetime.datetime.now())
#metadata['version'] = VERSION
#metadata['argv'] = " ".join(sys.argv).encode('ascii', errors='ignore').decode('ascii')
#metadata['python'] = " ".join(sys.version.splitlines()).encode('ascii', errors='ignore').decode('ascii')
#metadata['system'] = " ".join(os.uname())
# SAVE THE IMAGE ##########################################
output_file_path_template = "{}_TEL{:03d}_EV{:05d}.fits"
if output_directory is not None:
simtel_basename = os.path.basename(simtel_file_path)
prefix = os.path.join(output_directory,
simtel_basename)
else:
prefix = simtel_file_path
output_file_path = output_file_path_template.format(
prefix, tel_id, event_id)
print("saving", output_file_path)
images.save_benchmark_images(
img=calibrated_image_2d,
pe_img=pe_image_2d,
adc_sums_img=uncalibrated_image_2d,
pedestal_img=pedestal_2d,
gains_img=gains_2d,
pixel_pos=pixel_pos_2d,
pixel_mask=pixel_mask,
metadata=metadata,
output_file_path=output_file_path)
def extract_images(simtel_file_path,
tel_id_filter_list=None,
event_id_filter_list=None,
output_directory=None,
crop=True):
# EXTRACT IMAGES ##########################################################
# hessio_event_source returns a Python generator that streams data from an
# EventIO/HESSIO MC data file (e.g. a standard CTA data file).
# This generator contains ctapipe.core.Container instances ("event").
#
# Parameters:
# - max_events: maximum number of events to read
# - allowed_tels: select only a subset of telescope, if None, all are read.
source = hessio_event_source(simtel_file_path,
allowed_tels=tel_id_filter_list)
# ITERATE OVER EVENTS #####################################################
calib = CameraCalibrator(None, None)
for event in source:
calib.calibrate(event) # calibrate the event
event_id = int(event.dl0.event_id)
if (event_id_filter_list is None) or (event_id
in event_id_filter_list):
#print("event", event_id)
# ITERATE OVER IMAGES #############################################
for tel_id in event.trig.tels_with_trigger:
tel_id = int(tel_id)
if tel_id in tel_id_filter_list:
#print("telescope", tel_id)
# CHECK THE IMAGE GEOMETRY (ASTRI ONLY) ###################
# TODO
#print("checking geometry")
x, y = event.inst.subarray.tel[
tel_id].camera.pix_x, event.inst.subarray.tel[
tel_id].camera.pix_y
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
geom = CameraGeometry.guess(x, y, foclen)
if (geom.pix_type != "rectangular") or (geom.cam_id
not in ("ASTRICam",
"ASTRI")):
raise ValueError(
"Telescope {}: error (the input image is not a valide ASTRI telescope image) -> {} ({})"
.format(tel_id, geom.pix_type, geom.cam_id))
# GET IMAGES ##############################################
pe_image = event.mc.tel[
tel_id].photo_electron_image # 1D np array
#uncalibrated_image = event.dl0.tel[tel_id].adc_sums # ctapipe 0.3.0
uncalibrated_image = event.r0.tel[
tel_id].adc_sums # ctapipe 0.4.0
pedestal = event.mc.tel[tel_id].pedestal
gain = event.mc.tel[tel_id].dc_to_pe
pixel_pos = (event.inst.subarray.tel[tel_id].camera.pix_x,
event.inst.subarray.tel[tel_id].camera.pix_y)
calibrated_image = event.dl1.tel[tel_id].image
calibrated_image[1, calibrated_image[
0, :] <= ASTRI_CAM_CHANNEL_THRESHOLD] = 0
calibrated_image[0, calibrated_image[
0, :] > ASTRI_CAM_CHANNEL_THRESHOLD] = 0
calibrated_image = calibrated_image.sum(axis=0)
#print(pe_image.shape)
#print(calibrated_image.shape)
#print(uncalibrated_image.shape)
#print(pedestal.shape)
#print(gain.shape)
#print(pixel_pos.shape)
#print(pixel_pos[0])
#print(pixel_pos[1])
# CONVERTING GEOMETRY (1D TO 2D) ##########################
pe_image_2d = geometry_converter.astri_to_2d_array(
pe_image, crop=crop)
calibrated_image_2d = geometry_converter.astri_to_2d_array(
calibrated_image, crop=crop)
uncalibrated_image_2d = geometry_converter.astri_to_3d_array(
uncalibrated_image, crop=crop)
pedestal_2d = geometry_converter.astri_to_3d_array(
pedestal, crop=crop)
gains_2d = geometry_converter.astri_to_3d_array(gain,
crop=crop)
pixel_pos_2d = geometry_converter.astri_to_3d_array(
pixel_pos, crop=crop)
#print(pe_image_2d.shape)
#print(calibrated_image_2d.shape)
#print(uncalibrated_image_2d.shape)
#print(pedestal_2d.shape)
#print(gains_2d.shape)
#print(pixel_pos_2d.shape)
#sys.exit(0)
# GET PIXEL MASK ##########################################
pixel_mask = geometry_converter.astri_pixel_mask(
crop
) # 1 for pixels with actual data, 0 for virtual (blank) pixels
# MAKE METADATA ###########################################
metadata = {}
metadata[
'version'] = 1 # Version of the pywicta fits format
if crop:
metadata['cam_id'] = "ASTRI_CROPPED"
else:
metadata['cam_id'] = "ASTRI"
metadata['tel_id'] = tel_id
metadata['event_id'] = event_id
metadata['simtel'] = simtel_file_path
metadata['tel_trig'] = len(event.trig.tels_with_trigger)
metadata['energy'] = quantity_to_tuple(
event.mc.energy, 'TeV')
metadata['mc_az'] = quantity_to_tuple(event.mc.az, 'rad')
metadata['mc_alt'] = quantity_to_tuple(event.mc.alt, 'rad')
metadata['mc_corex'] = quantity_to_tuple(
event.mc.core_x, 'm')
metadata['mc_corey'] = quantity_to_tuple(
event.mc.core_y, 'm')
metadata['mc_hfi'] = quantity_to_tuple(
event.mc.h_first_int, 'm')
metadata['count'] = int(event.count)
metadata['run_id'] = int(event.dl0.obs_id)
metadata['tel_data'] = len(event.dl0.tels_with_data)
metadata['foclen'] = quantity_to_tuple(
event.inst.subarray.tel[tel_id].optics.
equivalent_focal_length, 'm')
metadata['tel_posx'] = quantity_to_tuple(
event.inst.subarray.tel_coords[tel_id].x, 'm')
metadata['tel_posy'] = quantity_to_tuple(
event.inst.subarray.tel_coords[tel_id].y, 'm')
metadata['tel_posz'] = quantity_to_tuple(
event.inst.subarray.tel_coords[tel_id].z, 'm')
# TODO: Astropy fails to store the following data in FITS files
#metadata['uid'] = os.getuid()
#metadata['datetime'] = str(datetime.datetime.now())
#metadata['version'] = VERSION
#metadata['argv'] = " ".join(sys.argv).encode('ascii', errors='ignore').decode('ascii')
#metadata['python'] = " ".join(sys.version.splitlines()).encode('ascii', errors='ignore').decode('ascii')
#metadata['system'] = " ".join(os.uname())
# SAVE THE IMAGE ##########################################
output_file_path_template = "{}_TEL{:03d}_EV{:05d}.fits"
if output_directory is not None:
simtel_basename = os.path.basename(simtel_file_path)
prefix = os.path.join(output_directory,
simtel_basename)
else:
prefix = simtel_file_path
output_file_path = output_file_path_template.format(
prefix, tel_id, event_id)
print("saving", output_file_path)
images.save_benchmark_images(
img=calibrated_image_2d,
pe_img=pe_image_2d,
adc_sums_img=uncalibrated_image_2d,
pedestal_img=pedestal_2d,
gains_img=gains_2d,
pixel_pos=pixel_pos_2d,
pixel_mask=pixel_mask,
metadata=metadata,
output_file_path=output_file_path)