do_remove_saturation = True
do_remove_RFI = True
positive_saturation = 2046
negative_saturation = -2047
saturation_post_removal_length = 50
saturation_half_hann_length = 50
polarization_flips = "polarization_flips.txt"
bad_antennas = "bad_antennas.txt"
additional_antenna_delays = "ant_delays.txt"
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder +
'/' +
additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = MultiFile_Dal1(raw_fpaths[station],
polarization_flips=polarization_flips,
bad_antennas=bad_antennas,
additional_ant_delays=additional_antenna_delays)
RFI_filter = window_and_filter(timeID=timeID, sname=station)
data = np.empty(block_size, dtype=np.double)
ant_names = TBB_data.get_antenna_names()
num_antenna_pairs = int(len(ant_names) / 2)
def save_EventByLoc(timeID, XYZT, station_timing_delays, pulse_index, output_folder, pulse_width=50, min_ant_amp=5, upsample_factor=0,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(logging_folder + "/log_out_"+str(pulse_index)+"_.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("saving traces to file", pulse_index)
print("location:", XYZT)
print("upsample_factor:", upsample_factor)
print()
print()
print("station timing offsets")
print(station_timing_delays)
print()
print()
print("pol flips")
print(polarization_flips)
print()
print()
print("bad antennas")
print(bad_antennas)
print()
print()
print("additional antenna delays")
print(additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays,
station_delay=station_timing_delays[sname]) for sname,fpaths in raw_fpaths.items() if sname in station_timing_delays}
data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_timing_delays}
trace_locator = getTrace_fromLoc( raw_data_files, data_filters )
print()
print()
print("data opened")
out_fname = data_dir + "/potSource_"+str(pulse_index)+".h5"
out_file = h5py.File(out_fname, "w")
for sname in station_timing_delays.keys():
TBB_file = raw_data_files[ sname ]
TBB_file.find_and_set_polarization_delay()
antenna_names = TBB_file.get_antenna_names()
h5_statGroup = out_file.create_group( sname )
print()
print(sname)
for even_ant_i in range(0,len(antenna_names),2):
even_ant_name = antenna_names[ even_ant_i ]
odd_ant_name = antenna_names[ even_ant_i+1 ]
starting_index, PolE_offset, throw, PolE_trace = trace_locator.get_trace_fromLoc(XYZT, even_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
throw, PolO_offset, throw, PolO_trace = trace_locator.get_trace_fromIndex(starting_index, odd_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
PolE_HE = np.abs(PolE_trace)
PolO_HE = np.abs(PolO_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(even_ant_name, (4, pulse_width ), dtype=np.double)
h5_Ant_dataset[0] = np.real(PolE_trace)
h5_Ant_dataset[1] = PolE_HE
h5_Ant_dataset[2] = np.real(PolO_trace)
h5_Ant_dataset[3] = PolO_HE
### note that peak time should NOT account for station timing offsets!
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs['PolE_timeOffset'] = -(PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset'] = -(PolO_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolE_timeOffset_CS'] = (PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset_CS'] = (PolO_offset - station_timing_delays[sname])
sample_time = 5.0E-9
if upsample_factor > 1:
PolE_HE = resample(PolE_HE, len(PolE_HE)*upsample_factor )
PolO_HE = resample(PolO_HE, len(PolO_HE)*upsample_factor )
sample_time /= upsample_factor
if np.max(PolE_HE)> min_ant_amp:
PolE_peak_finder = parabolic_fit( PolE_HE )
h5_Ant_dataset.attrs['PolE_peakTime'] = starting_index*5.0E-9 - (PolE_offset-station_timing_delays[sname]) + PolE_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(PolO_HE)> min_ant_amp:
PolO_peak_finder = parabolic_fit( PolO_HE )
h5_Ant_dataset.attrs['PolO_peakTime'] = starting_index*5.0E-9 - (PolO_offset-station_timing_delays[sname]) + PolO_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", even_ant_name, h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'])
out_file.close()
def open_files(self, station, log_func=do_nothing, force_metadata_ant_pos=True):
processed_data_loc = processed_data_dir(self.timeID)
#### open callibration data files ####
### TODO: fix these so they are accounted for in a more standard maner
self.additional_ant_delays = read_antenna_delays( processed_data_loc+'/'+self.additional_antenna_delays_fname )
self.bad_antennas = read_bad_antennas( processed_data_loc+'/'+self.bad_antennas_fname )
self.pol_flips = read_antenna_pol_flips( processed_data_loc+'/'+self.pol_flips_fname )
#### open data files, and find RFI ####
self.station = station
raw_fpaths = filePaths_by_stationName(self.timeID)[station]
self.input_file = MultiFile_Dal1( raw_fpaths, force_metadata_ant_pos=force_metadata_ant_pos )
self.input_file.set_polarization_flips( self.pol_flips )
if self.do_RFI_filtering and self.use_saved_RFI_info:
self.RFI_filters.append( window_and_filter(timeID=self.timeID, sname=station) )
elif self.do_RFI_filtering:
RFI_result = FindRFI(self.input_file, self.block_size, self.initial_RFI_block, self.RFI_num_blocks, self.RFI_max_blocks, verbose=False, figure_location=None)
self.RFI_filters.append( window_and_filter(find_RFI=RFI_result) )
else: ## only basic filtering
self.RFI_filters.append( window_and_filter(blocksize=self.block_size) )
#### find antenna pairs ####
self.antennas_to_use = pairData_evenAnts(self.input_file, self.bad_antennas, self.max_num_antennas )
self.num_antennas = len(self.antennas_to_use)
#### get antenna locations and delays ####
self.antenna_locations = np.zeros((self.num_antennas, 3), dtype=np.double)
self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double)
for ant_i, station_ant_i in enumerate(self.antennas_to_use):
ant_name = self.input_file.get_antenna_names()[ station_ant_i ]
self.antenna_locations[ant_i] = self.input_file.get_LOFAR_centered_positions()[ station_ant_i ]
self.antenna_delays[ant_i] = self.input_file.get_timing_callibration_delays()[ station_ant_i ]
## add additional timing delays
##note this only works for even antennas!
if ant_name in self.additional_ant_delays:
self.antenna_delays[ant_i] += self.additional_ant_delays[ ant_name ][0]
##### find window offsets and lengths ####
self.half_antenna_data_length = np.zeros(self.num_antennas, dtype=np.long)
for ant_i, station_ant_i in enumerate(self.antennas_to_use):
### find max duration to any of the prefered antennas
max_distance = 0.0
for ant_j, station_ant_j in enumerate(self.antennas_to_use):
if ant_j == ant_i:
continue
distance = np.linalg.norm( self.antenna_locations[ ant_j ]-self.antenna_locations[ant_i] )
if distance > max_distance:
max_distance = distance
self.half_antenna_data_length[ant_i] = int(self.pulse_length/2) + int(max_distance/(v_air*5.0E-9))
self.block_exclusion_length = int(0.1*self.block_size) + np.max( self.half_antenna_data_length ) + 1
self.active_block_length = self.block_size - 2*self.block_exclusion_length##the length of block used for looking at data
self.trace_length = 2*np.max(self.half_antenna_data_length )
self.trace_length = 2**( int(np.log2( self.trace_length )) + 1 )
#### allocate some memory ####
self.data_block = np.empty((self.num_antennas,self.block_size), dtype=np.complex)
self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double)
self.stage_1_imager = II_tools.image_data(self.antenna_locations, self.antenna_delays, self.trace_length, self.upsample_factor)
def save_Pulse(timeID, output_folder, event_index, pulse_time, station_delays, skip_stations=[], window_width=7E-6, pulse_width=50, block_size=2**16,
min_ant_amp=5.0, guess_flash_location=None, referance_station="CS002", polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
"""Desined to be used in conjuction with plot_multiple_data_all_stations.py. Given a pulse_time, and window_width, it saves a pulse on every antenna that is
centered on the largest peak in the window, with a width of pulse_width. station_delays doesn't need to be accurate, just should be the same as used in plot_multiple_data_all_stations.
Same for referance_station, and guess_flash_location. Note that this will save the pulse under an index, and will overwrite any other pulse saved under that index in the output folder"""
if referance_station in station_delays:
ref_delay = station_delays[referance_station]
station_delays = {sname:delay-ref_delay for sname,delay in station_delays.items()}
#### setup directory variables ####
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log.set(logging_folder + "/log_event_"+str(event_index)+".txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("pulse time:", pulse_time)
print("window_width:", window_width)
print("pulse_width:", pulse_width)
print("skip stations:", skip_stations)
print("guess_flash_location:", guess_flash_location)
print("input delays:")
print( station_delays)
print()
##station data
print()
print("opening station data")
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_delays.keys()}
data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_delays.keys()}
trace_locator = getTrace_fromLoc( raw_data_files, data_filters, {sname:0.0 for sname in station_delays.keys()} )
if guess_flash_location is not None:
guess_flash_location = np.array(guess_flash_location)
ref_stat_file = raw_data_files[ referance_station ]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in station_delays:
station_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
station_delays[sname] += (np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_delay)
station_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9
print()
print("used apparent delays:")
print(station_delays)
print()
out_fname = data_dir + "/potSource_"+str(event_index)+".h5"
out_file = h5py.File(out_fname, "w")
half_pulse_width = int(0.5*pulse_width)
window_width_samples = int(window_width/5.0E-9)
for sname in station_delays.keys():
if (not np.isfinite(station_delays[sname])) or sname in skip_stations:
continue
h5_statGroup = out_file.create_group( sname )
antenna_names = raw_data_files[sname].get_antenna_names()
data_arrival_index = int((pulse_time + station_delays[sname] - window_width*0.5)/5.0E-9)
print()
print(sname)
for even_ant_i in range(0, len(antenna_names), 2):
#### get window and find peak ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i], window_width_samples)
throw, odd_total_time_offset, throw, odd_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i+1], window_width_samples)
even_HE = np.abs(even_trace)
even_good = False
if np.max( even_HE ) > min_ant_amp:
even_good = True
even_para_fit = parabolic_fit( even_HE )
odd_HE = np.abs(odd_trace)
odd_good = False
if np.max( odd_HE ) > min_ant_amp:
odd_good = True
odd_para_fit = parabolic_fit( odd_HE )
if (not even_good) and (not odd_good):
continue
elif even_good and (not odd_good):
center_index = int(even_para_fit.peak_index)
elif (not even_good) and odd_good:
center_index = int(odd_para_fit.peak_index)
else: ## both good
even_amp = even_HE[ int(even_para_fit.peak_index) ]
odd_amp = odd_HE[ int(odd_para_fit.peak_index) ]
if even_amp > odd_amp:
center_index = int(even_para_fit.peak_index)
else:
center_index = int(odd_para_fit.peak_index)
#### now get data centered on peak. re-get data, as peak may be near edge ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i], half_pulse_width*2)
throw, odd_total_time_offset , throw, odd_trace = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i+1], half_pulse_width*2)
# even_trace = even_trace[center_index-half_pulse_width:center_index+half_pulse_width]
# odd_trace = odd_trace[center_index-half_pulse_width:center_index+half_pulse_width]
even_HE = np.abs(even_trace)
odd_HE = np.abs(odd_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(antenna_names[even_ant_i], (4, half_pulse_width*2), dtype=np.double)
h5_Ant_dataset[0] = np.real( even_trace )
h5_Ant_dataset[1] = even_HE
h5_Ant_dataset[2] = np.real( odd_trace )
h5_Ant_dataset[3] = odd_HE
WARNING: not sure this is correct. even_total_time_offset includes station delay
starting_index = data_arrival_index+center_index-half_pulse_width
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs['PolE_timeOffset'] = -even_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
h5_Ant_dataset.attrs['PolO_timeOffset'] = -odd_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
if np.max(even_HE) > min_ant_amp:
# even_HE = resample(even_HE, len(even_HE)*8 )
# even_para_fit = parabolic_fit( even_HE )
# h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - even_total_time_offset
even_para_fit = parabolic_fit( even_HE )
h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index + starting_index)*5.0E-9 - even_total_time_offset
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(odd_HE) > min_ant_amp:
# odd_HE = resample(odd_HE, len(odd_HE)*8 )
# odd_para_fit = parabolic_fit( odd_HE )
# h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - odd_total_time_offset
odd_para_fit = parabolic_fit( odd_HE )
h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index + starting_index)*5.0E-9 - odd_total_time_offset
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", antenna_names[even_ant_i], (data_arrival_index+center_index)*5.0E-9 - station_delays[sname], h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'] )
out_file.close()