def __init__(self,
TBB_datafile,
RFI_filter,
good_sources,
noise_block_range,
pulse_width=31E-9,
max_fraction_doubleZeros=0.0005):
self.TBB_datafile = TBB_datafile
self.RFI_filter = RFI_filter
self.blocksize = self.RFI_filter.blocksize
self.pulse_width = pulse_width
self.edge_width = int(RFI_filter.blocksize *
RFI_filter.half_window_percent)
self.good_sources = good_sources
self.max_fraction_doubleZeros = max_fraction_doubleZeros
self.antenna_names = TBB_datafile.get_antenna_names()
self.num_antennas = len(self.antenna_names)
self.num_pairs = int(len(self.antenna_names) / 2)
self.station_name = TBB_datafile.StationName
self.TraceLocator = getTrace_fromLoc({self.station_name: TBB_datafile},
{self.station_name: RFI_filter})
self.antenna_noise_power = [
get_noise_power(TBB_datafile, RFI_filter, anti, noise_block_range,
max_fraction_doubleZeros)
for anti in range(self.num_antennas)
]
self.event_samples = [[
self.TraceLocator.source_recieved_index(source.XYZT, ant_name)
for ant_name in self.antenna_names
] for source in self.good_sources]
tqdm.write("Loading data filters:")
pbar1 = tqdm(station_timing_offsets,
ascii=True,
unit_scale=True,
dynamic_ncols=True,
position=0)
data_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in pbar1
}
pbar1.close()
sorted_snames = natural_sort(station_timing_offsets.keys())
trace_locator = getTrace_fromLoc(TBB_data, data_filters,
station_timing_offsets)
#############################################
## IMPORT SOURCE INFO FROM JSON FILE ##
#############################################
width = 40 #set width of datablock
cal = calibrate(timeID, width, verbose=False)
stokes = stokes_params()
pName = "KC13" #phenomena name
source_info_loc = processed_data_dir(
timeID) + "/polarization_data/Lightning Phenomena/K changes"
with open(source_info_loc + '/' + "source_info_{}.json".format(pName),
def save_EventByLoc(timeID, XYZT, station_timing_delays, pulse_index, output_folder, pulse_width=50, min_ant_amp=5, upsample_factor=0,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(logging_folder + "/log_out_"+str(pulse_index)+"_.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("saving traces to file", pulse_index)
print("location:", XYZT)
print("upsample_factor:", upsample_factor)
print()
print()
print("station timing offsets")
print(station_timing_delays)
print()
print()
print("pol flips")
print(polarization_flips)
print()
print()
print("bad antennas")
print(bad_antennas)
print()
print()
print("additional antenna delays")
print(additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays,
station_delay=station_timing_delays[sname]) for sname,fpaths in raw_fpaths.items() if sname in station_timing_delays}
data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_timing_delays}
trace_locator = getTrace_fromLoc( raw_data_files, data_filters )
print()
print()
print("data opened")
out_fname = data_dir + "/potSource_"+str(pulse_index)+".h5"
out_file = h5py.File(out_fname, "w")
for sname in station_timing_delays.keys():
TBB_file = raw_data_files[ sname ]
TBB_file.find_and_set_polarization_delay()
antenna_names = TBB_file.get_antenna_names()
h5_statGroup = out_file.create_group( sname )
print()
print(sname)
for even_ant_i in range(0,len(antenna_names),2):
even_ant_name = antenna_names[ even_ant_i ]
odd_ant_name = antenna_names[ even_ant_i+1 ]
starting_index, PolE_offset, throw, PolE_trace = trace_locator.get_trace_fromLoc(XYZT, even_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
throw, PolO_offset, throw, PolO_trace = trace_locator.get_trace_fromIndex(starting_index, odd_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
PolE_HE = np.abs(PolE_trace)
PolO_HE = np.abs(PolO_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(even_ant_name, (4, pulse_width ), dtype=np.double)
h5_Ant_dataset[0] = np.real(PolE_trace)
h5_Ant_dataset[1] = PolE_HE
h5_Ant_dataset[2] = np.real(PolO_trace)
h5_Ant_dataset[3] = PolO_HE
### note that peak time should NOT account for station timing offsets!
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs['PolE_timeOffset'] = -(PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset'] = -(PolO_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolE_timeOffset_CS'] = (PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset_CS'] = (PolO_offset - station_timing_delays[sname])
sample_time = 5.0E-9
if upsample_factor > 1:
PolE_HE = resample(PolE_HE, len(PolE_HE)*upsample_factor )
PolO_HE = resample(PolO_HE, len(PolO_HE)*upsample_factor )
sample_time /= upsample_factor
if np.max(PolE_HE)> min_ant_amp:
PolE_peak_finder = parabolic_fit( PolE_HE )
h5_Ant_dataset.attrs['PolE_peakTime'] = starting_index*5.0E-9 - (PolE_offset-station_timing_delays[sname]) + PolE_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(PolO_HE)> min_ant_amp:
PolO_peak_finder = parabolic_fit( PolO_HE )
h5_Ant_dataset.attrs['PolO_peakTime'] = starting_index*5.0E-9 - (PolO_offset-station_timing_delays[sname]) + PolO_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", even_ant_name, h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'])
out_file.close()
additional_antenna_delays)
station_timing_offsets = read_station_delays(processed_data_folder + '/' +
station_delay_file)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname,fpaths in raw_fpaths.items() if sname in station_timing_offsets}
sorted_station_names = natural_sort(raw_data_files.keys())
data_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in station_timing_offsets
}
trace_locator = getTrace_fromLoc(raw_data_files, data_filters,
station_timing_offsets)
plt.figure(figsize=(20, 10))
H = 0
for sname in sorted_station_names:
input_file = raw_data_files[sname]
antenna_names = input_file.get_antenna_names()[::2] ## even antennas
antenna_locations = input_file.get_LOFAR_centered_positions()[::2]
station_location_difference = source_XYZT[:3] - np.average(
antenna_locations, axis=0)
zenith = np.arctan2(station_location_difference[2],
np.linalg.norm(station_location_difference[:2]))
sys.stdout = open(
processed_data_folder + "/polarization_data/Pulse_Plots/" +
"{}.log".format(srcName), 'a')
def save_Pulse(timeID, output_folder, event_index, pulse_time, station_delays, skip_stations=[], window_width=7E-6, pulse_width=50, block_size=2**16,
min_ant_amp=5.0, guess_flash_location=None, referance_station="CS002", polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
"""Desined to be used in conjuction with plot_multiple_data_all_stations.py. Given a pulse_time, and window_width, it saves a pulse on every antenna that is
centered on the largest peak in the window, with a width of pulse_width. station_delays doesn't need to be accurate, just should be the same as used in plot_multiple_data_all_stations.
Same for referance_station, and guess_flash_location. Note that this will save the pulse under an index, and will overwrite any other pulse saved under that index in the output folder"""
if referance_station in station_delays:
ref_delay = station_delays[referance_station]
station_delays = {sname:delay-ref_delay for sname,delay in station_delays.items()}
#### setup directory variables ####
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log.set(logging_folder + "/log_event_"+str(event_index)+".txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("pulse time:", pulse_time)
print("window_width:", window_width)
print("pulse_width:", pulse_width)
print("skip stations:", skip_stations)
print("guess_flash_location:", guess_flash_location)
print("input delays:")
print( station_delays)
print()
##station data
print()
print("opening station data")
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_delays.keys()}
data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_delays.keys()}
trace_locator = getTrace_fromLoc( raw_data_files, data_filters, {sname:0.0 for sname in station_delays.keys()} )
if guess_flash_location is not None:
guess_flash_location = np.array(guess_flash_location)
ref_stat_file = raw_data_files[ referance_station ]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in station_delays:
station_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
station_delays[sname] += (np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_delay)
station_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9
print()
print("used apparent delays:")
print(station_delays)
print()
out_fname = data_dir + "/potSource_"+str(event_index)+".h5"
out_file = h5py.File(out_fname, "w")
half_pulse_width = int(0.5*pulse_width)
window_width_samples = int(window_width/5.0E-9)
for sname in station_delays.keys():
if (not np.isfinite(station_delays[sname])) or sname in skip_stations:
continue
h5_statGroup = out_file.create_group( sname )
antenna_names = raw_data_files[sname].get_antenna_names()
data_arrival_index = int((pulse_time + station_delays[sname] - window_width*0.5)/5.0E-9)
print()
print(sname)
for even_ant_i in range(0, len(antenna_names), 2):
#### get window and find peak ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i], window_width_samples)
throw, odd_total_time_offset, throw, odd_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i+1], window_width_samples)
even_HE = np.abs(even_trace)
even_good = False
if np.max( even_HE ) > min_ant_amp:
even_good = True
even_para_fit = parabolic_fit( even_HE )
odd_HE = np.abs(odd_trace)
odd_good = False
if np.max( odd_HE ) > min_ant_amp:
odd_good = True
odd_para_fit = parabolic_fit( odd_HE )
if (not even_good) and (not odd_good):
continue
elif even_good and (not odd_good):
center_index = int(even_para_fit.peak_index)
elif (not even_good) and odd_good:
center_index = int(odd_para_fit.peak_index)
else: ## both good
even_amp = even_HE[ int(even_para_fit.peak_index) ]
odd_amp = odd_HE[ int(odd_para_fit.peak_index) ]
if even_amp > odd_amp:
center_index = int(even_para_fit.peak_index)
else:
center_index = int(odd_para_fit.peak_index)
#### now get data centered on peak. re-get data, as peak may be near edge ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i], half_pulse_width*2)
throw, odd_total_time_offset , throw, odd_trace = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i+1], half_pulse_width*2)
# even_trace = even_trace[center_index-half_pulse_width:center_index+half_pulse_width]
# odd_trace = odd_trace[center_index-half_pulse_width:center_index+half_pulse_width]
even_HE = np.abs(even_trace)
odd_HE = np.abs(odd_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(antenna_names[even_ant_i], (4, half_pulse_width*2), dtype=np.double)
h5_Ant_dataset[0] = np.real( even_trace )
h5_Ant_dataset[1] = even_HE
h5_Ant_dataset[2] = np.real( odd_trace )
h5_Ant_dataset[3] = odd_HE
WARNING: not sure this is correct. even_total_time_offset includes station delay
starting_index = data_arrival_index+center_index-half_pulse_width
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs['PolE_timeOffset'] = -even_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
h5_Ant_dataset.attrs['PolO_timeOffset'] = -odd_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
if np.max(even_HE) > min_ant_amp:
# even_HE = resample(even_HE, len(even_HE)*8 )
# even_para_fit = parabolic_fit( even_HE )
# h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - even_total_time_offset
even_para_fit = parabolic_fit( even_HE )
h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index + starting_index)*5.0E-9 - even_total_time_offset
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(odd_HE) > min_ant_amp:
# odd_HE = resample(odd_HE, len(odd_HE)*8 )
# odd_para_fit = parabolic_fit( odd_HE )
# h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - odd_total_time_offset
odd_para_fit = parabolic_fit( odd_HE )
h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index + starting_index)*5.0E-9 - odd_total_time_offset
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", antenna_names[even_ant_i], (data_arrival_index+center_index)*5.0E-9 - station_delays[sname], h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'] )
out_file.close()