def get_trace_fromLoc(self,
XYZT,
ant_name,
width,
do_remove_RFI=True,
do_remove_saturation=True,
positive_saturation=2046,
negative_saturation=-2047,
removal_length=50,
half_hann_length=50):
"""given the location of the source in XYZT, name of the antenna, and width (in num data samples) of the desired pulse,
return starting_index of the returned trace, total time offest of that antenna, predicted arrival time of pulse at that antenna,
and the time trace centered on the arrival time."""
station_name = SId_to_Sname[int(ant_name[:3])]
station_data = self.data_file_dict[station_name]
data_filter = self.data_filter_dict[station_name]
file_antenna_index = station_data.get_antenna_names().index(ant_name)
# ant_loc = station_data.get_LOFAR_centered_positions()[ file_antenna_index ]
# ant_time_offset = station_data.get_timing_callibration_delays()[file_antenna_index] - station_data.get_nominal_sample_number()*5.0E-9
# total_time_offset = ant_time_offset + self.station_timing_calibration[station_name]
# predicted_arrival_time = np.linalg.norm( XYZT[:3]-ant_loc )/v_air + XYZT[3]
total_time_offset = station_data.get_total_delays()[file_antenna_index]
antenna_locations = station_data.get_LOFAR_centered_positions()
predicted_arrival_time = station_data.get_geometric_delays(
XYZT[:3],
antenna_locations=antenna_locations[
file_antenna_index:file_antenna_index + 1]) + XYZT[3]
data_arrival_index = int(predicted_arrival_time / 5.0E-9 +
total_time_offset / 5.0E-9)
local_data_index = int(data_filter.blocksize * 0.5)
data_start_sample = data_arrival_index - local_data_index
input_data = station_data.get_data(data_start_sample,
data_filter.blocksize,
antenna_index=file_antenna_index)
input_data = np.array(input_data, dtype=np.double)
if do_remove_saturation:
remove_saturation(input_data, positive_saturation,
negative_saturation, removal_length,
half_hann_length)
width_before = int(width * 0.5)
width_after = width - width_before
if do_remove_RFI:
data = data_filter.filter(
input_data)[local_data_index - width_before:local_data_index +
width_after]
else:
data = input_data[local_data_index -
width_before:local_data_index + width_after]
return data_start_sample + local_data_index - width_before, total_time_offset, predicted_arrival_time, data
def get_trace_fromIndex(self,
starting_index,
ant_name,
width,
do_remove_RFI=True,
do_remove_saturation=True,
positive_saturation=2046,
negative_saturation=-2047,
removal_length=50,
saturation_half_hann_length=50):
"""similar to previous, but now retrieve trace based on location in file. For repeatability.
Has same returns. predicted arrival time is just the time in middle of trace"""
station_name = SId_to_Sname[int(ant_name[:3])]
station_data = self.data_file_dict[station_name]
data_filter = self.data_filter_dict[station_name]
file_antenna_index = station_data.get_antenna_names().index(ant_name)
total_time_offset = station_data.get_total_delays()[file_antenna_index]
local_data_index = int(data_filter.blocksize * 0.5)
input_data = station_data.get_data(starting_index - local_data_index,
data_filter.blocksize,
antenna_index=file_antenna_index)
input_data = np.array(input_data, dtype=np.double)
if do_remove_saturation:
remove_saturation(input_data, positive_saturation,
negative_saturation, removal_length,
saturation_half_hann_length)
if self.return_dbl_zeros:
dbl_zeros = num_double_zeros(
input_data[local_data_index:local_data_index + width])
if do_remove_RFI:
data = data_filter.filter(
input_data)[local_data_index:local_data_index + width]
else:
data = input_data[local_data_index:local_data_index + width]
predicted_arrival_time = starting_index * 5.0E-9 - total_time_offset + 0.5 * width * 5.0E-9
if self.return_dbl_zeros:
return starting_index, total_time_offset, predicted_arrival_time, data, dbl_zeros
else:
return starting_index, total_time_offset, predicted_arrival_time, data
def open_antenna_SampleNumber(self, antenna_i, sample_number):
"""opens the data for a station, where every antenna starts at the given sample number. Returns the data block for this station, and the start times for each antenna"""
# print('opening:', antenna_i)
station_i = self.antI_to_station[ antenna_i ]
stationFile = self.station_data_files[ station_i ]
data_filter = self.RFI_filters[ station_i ]
station_antenna_index = self.station_antenna_index[ antenna_i ]
TMP = stationFile.get_data(sample_number, self.blocksize, antenna_index=station_antenna_index )
if len(TMP) != self.blocksize:
MSG = 'data length wrong. is: ' + str(len(TMP)) + " should be " + str(self.blocksize) + ". sample: " + \
str(sample_number) + ". antenna: " + str(station_antenna_index) + ". station: " + stationFile.get_station_name()
raise Exception( MSG )
self.data_loss_spans[ antenna_i ], DL = locate_data_loss(TMP, self.num_dataLoss_zeros)
self.tmp_workspace[:] = TMP
if self.remove_saturation:
self.staturation_removals[antenna_i] = remove_saturation( self.tmp_workspace, self.positive_saturation, self.negative_saturation, self.saturation_removal_length, self.saturation_half_hann_length )
else:
self.staturation_removals[antenna_i] = []
if self.remove_RFI:
self.raw_data[ antenna_i ] = data_filter.filter( self.tmp_workspace )
else:
self.raw_data[ antenna_i ] = self.tmp_workspace
self.starting_time[ antenna_i ] = sample_number*5E-9 - self.antenna_delays[ antenna_i ]
self.antenna_data_loaded[ antenna_i ] = True
return self.raw_data[ antenna_i ], self.starting_time[ antenna_i ]
bad_antennas=bad_antennas,
additional_ant_delays=additional_antenna_delays)
RFI_filter = window_and_filter(timeID=timeID, sname=station)
data = np.empty(block_size, dtype=np.double)
ant_names = TBB_data.get_antenna_names()
num_antenna_pairs = int(len(ant_names) / 2)
t0 = np.arange(block_size)
H = 0
T = t0 + point
for pair in range(num_antenna_pairs):
data[:] = TBB_data.get_data(point, block_size, antenna_index=pair * 2)
remove_saturation(data, positive_saturation, negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
filtered_data = RFI_filter.filter(data)
even_HE = np.abs(filtered_data)
data[:] = TBB_data.get_data(point,
block_size,
antenna_index=pair * 2 + 1)
remove_saturation(data, positive_saturation, negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
filtered_data = RFI_filter.filter(data)
odd_HE = np.abs(filtered_data)
peak = max(np.max(even_HE), np.max(odd_HE))
def process_block(self, start_index, block_index, h5_groupobject, log_func=do_nothing):
#### open and filter the data ####
prefered_ant_i = None
prefered_ant_dataLoss = np.inf
for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use):
data_file = self.input_files[station_i]
RFI_filter = self.RFI_filters[station_i]
offset = self.antenna_data_offsets[ant_i]
self.data_block[ant_i, :] = data_file.get_data(start_index+offset, self.block_size, antenna_index=station_ant_i) ## get the data. accounting for the offsets calculated earlier
remove_saturation(self.data_block[ant_i, :], self.positive_saturation, self.negative_saturation, post_removal_length=self.saturation_removal_length,
half_hann_length=self.saturation_hann_window_length)
if ant_i in self.station_to_antenna_indeces_dict[ self.prefered_station ]: ## this must be done before filtering
num_D_zeros = num_double_zeros( self.data_block[ant_i, :] )
if num_D_zeros < prefered_ant_dataLoss: ## this antenna could be teh antenna, in the prefered station, with least data loss
prefered_ant_dataLoss = num_D_zeros
prefered_ant_i = ant_i
self.data_block[ant_i, :] = RFI_filter.filter( self.data_block[ant_i, :] )
### make output object ###
h5_groupobject = h5_groupobject.create_group(str(block_index))
h5_groupobject.attrs['block_index'] = block_index
h5_groupobject.attrs['start_index'] = start_index
h5_groupobject.attrs['prefered_ant_i'] = prefered_ant_i
#### find sources ####
np.abs( self.data_block[ prefered_ant_i ], out=self.hilbert_envelope_tmp )
for event_i in range(self.max_events_perBlock):
## find peak ##
peak_loc = np.argmax(self.hilbert_envelope_tmp[self.startBlock_exclusion : -self.endBlock_exclusion ]) + self.startBlock_exclusion
trace_start_loc = peak_loc - int( self.pulse_length/2 ) # pobably account for fact that trace is now longer than pulse_length on prefered antenna
if self.hilbert_envelope_tmp[peak_loc] < self.min_pref_ant_amplitude:
log_func("peaks are too small. Done searching")
break
## select data for stage one ##
s1_ant_i = 0
for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use):
if self.use_core_stations_S1 or self.is_not_core[ant_i]:
half_window_length = self.half_antenna_data_length[ant_i]
self.stage_1_imager.set_data( self.data_block[ant_i][trace_start_loc : trace_start_loc+2*half_window_length], s1_ant_i )
s1_ant_i += 1
## fft and xcorrelation ##
self.stage_1_imager.prepare_image( prefered_ant_i, self.min_pulse_amplitude )
log_func("source:", event_i)
stage_1_result, num_itter, num_stage1_itters = stochastic_minimizer(self.stage_1_imager.intensity, self.bounding_box, converg_num=self.stage_1_converg_num,
converg_rad=self.stage_1_converg_radius, max_itters=self.stage_1_max_itters)
log_func(" stoch. itters:", num_itter, num_stage1_itters)
## select data for stage 2 ##
previous_solution = stage_1_result
converged = False
for stage2loop_i in range(self.stage_2_max_itters):
problem = False
s2_ant_i = -1
for ant_i in range( self.num_antennas ):
if self.use_core_stations_S2 or self.is_not_core[ant_i]:
s2_ant_i += 1 ## do this here cause of break below
modeled_dt = -( np.linalg.norm( self.antenna_locations[ prefered_ant_i ]-previous_solution.x ) -
np.linalg.norm( self.antenna_locations[ant_i]-previous_solution.x ) )/v_air
modeled_dt -= self.antenna_delays[ prefered_ant_i ] - self.antenna_delays[ant_i]
modeled_dt /= 5.0E-9
modeled_dt += peak_loc
modeled_dt = int(modeled_dt)
if modeled_dt+int(self.pulse_length/2) >= len(self.data_block[ant_i]):
problem = True
break
self.stage_2_imager.set_data( self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)]*self.stage_2_window, s2_ant_i, -modeled_dt*5.0E-9 )
if problem:
log_func("unknown problem. LOC at", previous_solution.x)
self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] = 0.0
continue
## fft and and xcorrelation ##
self.stage_2_imager.prepare_image( self.min_pulse_amplitude )
#
BB = np.array( [ [previous_solution.x[0]-50, previous_solution.x[0]+50], [previous_solution.x[1]-50, previous_solution.x[1]+50], [previous_solution.x[2]-50, previous_solution.x[2]+50] ] )
stage_2_result, s2_itternum, num_stage2_itters = stochastic_minimizer(self.stage_2_imager.intensity_ABSbefore , BB, converg_num=5, test_spot=previous_solution.x,
converg_rad=self.stage_2_convergence_length, max_itters=self.stage_2_max_stoch_itters, options={'maxiter':1000})
D = np.linalg.norm( stage_2_result.x - previous_solution.x )
log_func(" s2 itter: {:2d} {:4.2f} {:d}".format(stage2loop_i, -stage_2_result.fun, int(D)) )
if D < self.stage_2_convergence_length:
converged = True
break
elif D > self.stage_2_break_length:
converged = False
break
previous_solution = stage_2_result
new_stage_1_result = minimize(self.stage_1_imager.intensity, stage_2_result.x, method="Nelder-Mead", options={'maxiter':1000})
log_func(" old S1: {:4.2f} new SH1: {:4.2f}".format(-stage_1_result.fun, -new_stage_1_result.fun) )
if stage_1_result.fun < new_stage_1_result.fun:
S1_S2_distance = np.linalg.norm(stage_1_result.x-stage_2_result.x)
else:
S1_S2_distance = np.linalg.norm(new_stage_1_result.x-stage_2_result.x)
log_func(" loc: {:d} {:d} {:d}".format(int(stage_2_result.x[0]), int(stage_2_result.x[1]), int(stage_2_result.x[2])) )
log_func(" S1-S2 distance: {:d} converged: {} ".format( int(S1_S2_distance), converged) )
log_func(" intensity: {:4.2f} amplitude: {:d} ".format( -stage_2_result.fun, int(self.hilbert_envelope_tmp[peak_loc])) )
log_func()
log_func()
## save to file ##
source_dataset = h5_groupobject.create_dataset(str(event_i), (self.num_antennas,self.pulse_length), dtype=np.complex)
source_dataset.attrs["loc"] = stage_2_result.x
source_dataset.attrs["unique_index"] = block_index*self.max_events_perBlock + event_i
source_time_s2 = (peak_loc+start_index+self.antenna_data_offsets[prefered_ant_i])*5.0E-9 - np.linalg.norm( stage_2_result.x - self.antenna_locations[ prefered_ant_i ] )/v_air
source_time_s2 -= self.prefered_station_timing_offset + self.prefered_station_antenna_timing_offsets[ self.antennas_to_use[prefered_ant_i][1] ]
source_dataset.attrs["T"] = source_time_s2
source_dataset.attrs["peak_index"] = peak_loc
source_dataset.attrs["intensity"] = -stage_2_result.fun
source_dataset.attrs["stage_1_success"] = (num_stage1_itters==self.stage_1_converg_num)
source_dataset.attrs["stage_1_num_itters"] = num_stage1_itters
source_dataset.attrs["amplitude"] = self.hilbert_envelope_tmp[peak_loc]
source_dataset.attrs["S1_S2_distance"] = S1_S2_distance
source_dataset.attrs["converged"] = converged
source_time_s1 = (peak_loc+start_index+self.antenna_data_offsets[prefered_ant_i])*5.0E-9 - np.linalg.norm( stage_1_result.x - self.antenna_locations[ prefered_ant_i ] )/v_air
source_time_s1 -= self.prefered_station_timing_offset + self.prefered_station_antenna_timing_offsets[ self.antennas_to_use[prefered_ant_i][1] ]
source_dataset.attrs["XYZT_s1"] = np.append(stage_1_result.x, [source_time_s1])
#### erase the peaks !! ####
# self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] *= self.erasure_window
self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] = 0.0
if converged and self.erase_pulses:
for ant_i in range( self.num_antennas ):
modeled_dt = -( np.linalg.norm( self.antenna_locations[ prefered_ant_i ]-stage_2_result.x ) -
np.linalg.norm( self.antenna_locations[ant_i]-stage_2_result.x ) )/v_air
modeled_dt -= self.antenna_delays[ prefered_ant_i ] - self.antenna_delays[ant_i]
modeled_dt /= 5.0E-9
modeled_dt += peak_loc
modeled_dt = int(modeled_dt)
source_dataset[ant_i] = self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)]
self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)] *= self.erasure_window
def plot_blocks(timeID,
block_size,
block_starts,
guess_delays,
guess_location=None,
bad_stations=[],
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt",
do_remove_saturation=True,
do_remove_RFI=True,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=5,
referance_station="CS002"):
"""plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays,
if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero.
A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations"""
if referance_station in guess_delays:
ref_delay = guess_delays[referance_station]
guess_delays = {
sname: delay - ref_delay
for sname, delay in guess_delays.items()
}
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder +
'/' +
additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in raw_fpaths.keys() if sname not in bad_stations}
if guess_location is not None:
guess_location = np.array(guess_location)
ref_stat_file = raw_data_files[referance_station]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(
ant_loc - guess_location[:3]
) / v_air - ref_stat_file.get_nominal_sample_number() * 5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in guess_delays:
guess_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
guess_delays[sname] += (
np.linalg.norm(ant_loc - guess_location[:3]) / v_air -
ref_delay)
guess_delays[sname] -= data_file.get_nominal_sample_number(
) * 5.0E-9
RFI_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in raw_fpaths.keys() if sname not in bad_stations
}
data = np.empty(block_size, dtype=np.double)
height = 0
t0 = np.arange(block_size) * 5.0E-9
transform = blended_transform_factory(plt.gca().transAxes,
plt.gca().transData)
sname_X_loc = 0.0
for sname, data_file in raw_data_files.items():
print(sname)
station_delay = -data_file.get_nominal_sample_number() * 5.0E-9
if sname in guess_delays:
station_delay = guess_delays[sname]
station_delay_points = int(station_delay / 5.0E-9)
RFI_filter = RFI_filters[sname]
ant_names = data_file.get_antenna_names()
num_antenna_pairs = int(len(ant_names) / 2)
peak_height = 0.0
for point in block_starts:
T = t0 + point * 5.0E-9
for pair in range(num_antenna_pairs):
data[:] = data_file.get_data(point + station_delay_points,
block_size,
antenna_index=pair * 2)
if do_remove_saturation:
remove_saturation(data, positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter(data)
else:
filtered_data = hilbert(data)
even_HE = np.abs(filtered_data)
data[:] = data_file.get_data(point + station_delay_points,
block_size,
antenna_index=pair * 2 + 1)
if do_remove_saturation:
remove_saturation(data, positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter(data)
else:
filtered_data = hilbert(data)
odd_HE = np.abs(filtered_data)
plt.plot(T, even_HE + height, 'r')
plt.plot(T, odd_HE + height, 'g')
max_even = np.max(even_HE)
if max_even > peak_height:
peak_height = max_even
max_odd = np.max(odd_HE)
if max_odd > peak_height:
peak_height = max_odd
# plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
plt.annotate(sname, (sname_X_loc, height),
textcoords=transform,
xycoords=transform)
height += 2 * peak_height
plt.show()
def plot_a_station(self, sname, height):
print("plotting", sname)
station_clock_offset = self.clock_offset_manager.get_visual_offsets(
)[sname]
antenna_delays = self.antenna_time_corrections[sname]
TBB_file = self.TBB_files[sname]
RFI_filter = self.RFI_filters[sname]
initial_block = self.frame_manager.get_block()
#### open data, track saturation ####
saturated_ranges = {block_i: [] for block_i in range(self.num_blocks)}
station_max = 0.0
for even_ant_i in range(0, len(antenna_delays), 2):
delay = (antenna_delays[even_ant_i] +
antenna_delays[even_ant_i + 1]) * 0.5
total_delay = delay + station_clock_offset
delay_points = int(total_delay / 5.0E-9)
for block_i in range(self.num_blocks):
block_start = (block_i + initial_block) * self.block_size
##### even ####
self.temp_data_block[even_ant_i,
block_i, :] = TBB_file.get_data(
block_start + delay_points,
self.block_size,
antenna_index=even_ant_i)
if self.do_remove_saturation:
sat_ranges = remove_saturation(
self.temp_data_block[even_ant_i, block_i, :],
self.positive_saturation, self.negative_saturation,
self.saturation_post_removal_length,
self.saturation_half_hann_length)
saturated_ranges[block_i] += sat_ranges
if self.do_remove_RFI:
filtered_data = RFI_filter.filter(
self.temp_data_block[even_ant_i, block_i, :])
else:
filtered_data = hilbert(self.temp_data_block[even_ant_i,
block_i, :])
self.temp_data_block[even_ant_i,
block_i, :] = np.abs(filtered_data)
peak = np.max(self.temp_data_block[even_ant_i, block_i, :])
if peak > station_max:
station_max = peak
##### odd ####
self.temp_data_block[even_ant_i + 1,
block_i, :] = TBB_file.get_data(
block_start + delay_points,
self.block_size,
antenna_index=even_ant_i + 1)
if self.do_remove_saturation:
sat_ranges = remove_saturation(
self.temp_data_block[even_ant_i + 1, block_i, :],
self.positive_saturation, self.negative_saturation,
self.saturation_post_removal_length,
self.saturation_half_hann_length)
saturated_ranges[block_i] += sat_ranges
if self.do_remove_RFI:
filtered_data = RFI_filter.filter(
self.temp_data_block[even_ant_i + 1, block_i, :])
else:
filtered_data = hilbert(
self.temp_data_block[even_ant_i + 1, block_i, :])
self.temp_data_block[even_ant_i + 1,
block_i, :] = np.abs(filtered_data)
peak = np.max(self.temp_data_block[even_ant_i + 1, block_i, :])
if peak > station_max:
station_max = peak
#### plot saturated bits ####
time_by_block = {
block_i: (self.time_array +
(block_i + initial_block) * self.block_size * 5.0E-9)
for block_i in range(self.num_blocks)
}
for block_i, ranges in saturated_ranges.items():
T = time_by_block[block_i]
for imin, imax in ranges:
Tmin = T[imin]
width = T[imax - 1] - Tmin ## range does not include end point
rect = patches.Rectangle((Tmin, height),
width,
1,
linewidth=1,
edgecolor='r',
facecolor='r')
self.axes.add_patch(rect)
#### plot data ####
for pair_i in range(int(len(antenna_delays) / 2)):
even_ant = pair_i * 2
odd_odd = pair_i * 2 + 1
for block_i in range(self.num_blocks):
block_start = (block_i + initial_block) * self.block_size
even_trace = self.temp_data_block[even_ant, block_i]
odd_trace = self.temp_data_block[odd_odd, block_i]
even_trace /= station_max
odd_trace /= station_max
even_trace += height
odd_trace += height
T = time_by_block[block_i]
plt.plot(T, even_trace, 'g')
plt.plot(T, odd_trace, 'm')
minT, maxT = self.frame_manager.get_T_bounds()
self.pulse_manager.plot_lines(sname, height, minT, maxT,
station_clock_offset)
def __init__(self,
TBB_data,
polarization,
initial_block,
number_of_blocks,
pulses_per_block=10,
pulse_length=50,
min_amplitude=50,
upsample_factor=0,
min_num_antennas=4,
max_num_planewaves=np.inf,
timeID=None,
blocksize=2**16,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
verbose=True):
self.parabolic_fitter = parabolic_fitter()
self.polarization = polarization
self.TBB_data = TBB_data
left_pulse_length = int(pulse_length / 2)
right_pulse_length = pulse_length - left_pulse_length
ant_names = TBB_data.get_antenna_names()
num_antenna_pairs = int(len(ant_names) / 2)
self.num_found_planewaves = 0
if num_antenna_pairs < min_num_antennas:
return
RFI_filter = window_and_filter(
timeID=timeID,
sname=TBB_data.get_station_name(),
blocksize=(blocksize if timeID is None else None))
block_size = RFI_filter.blocksize
self.num_found_planewaves = 0
data = np.empty((num_antenna_pairs, block_size), dtype=np.double)
self.planewave_data = []
self.planewave_second_derivatives = []
for block in range(initial_block, initial_block + number_of_blocks):
if self.num_found_planewaves >= max_num_planewaves:
break
#### open and filter data
# print('open block', block)
for pair_i in range(num_antenna_pairs):
ant_i = pair_i * 2 + polarization
data[pair_i, :] = TBB_data.get_data(block * block_size,
block_size,
antenna_index=ant_i)
remove_saturation(data[pair_i, :], positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
np.abs(RFI_filter.filter(data[pair_i, :]), out=data[pair_i, :])
#### loop over finding planewaves
i = 0
while i < pulses_per_block:
if self.num_found_planewaves >= max_num_planewaves:
break
pulse_location = 0
pulse_amplitude = 0
## find highest peak
for pair_i, HE in enumerate(data):
loc = np.argmax(HE)
amp = HE[loc]
if amp > pulse_amplitude:
pulse_amplitude = amp
pulse_location = loc
## check if is strong enough
if pulse_amplitude < min_amplitude:
break
## get
newPW_data = np.full(num_antenna_pairs,
np.nan,
dtype=np.double)
newPW_derivatives = np.full(num_antenna_pairs,
np.nan,
dtype=np.double)
for pair_i, HE in enumerate(data):
# ant_i = pair_i*2 + polarization
signal = np.array(
HE[pulse_location - left_pulse_length:pulse_location +
right_pulse_length])
if num_double_zeros(signal, threshold=0.1) == 0:
sample_time = 5.0E-9
if upsample_factor > 1:
sample_time /= upsample_factor
signal = resample(signal,
len(signal) * upsample_factor)
newPW_data[pair_i] = self.parabolic_fitter.fit(
signal) * sample_time
newPW_derivatives[
pair_i] = self.parabolic_fitter.second_derivative(
) * sample_time
HE[pulse_location - left_pulse_length:pulse_location +
right_pulse_length] = 0.0
if np.sum(np.isfinite(newPW_data)) >= min_num_antennas:
self.planewave_data.append(newPW_data)
self.planewave_second_derivatives.append(newPW_derivatives)
self.num_found_planewaves += 1
i += 1
def more_plots(some_data,First,ax,text):
global l,m
ax.clear()
height = 0
t0 = np.arange(block_size)*5.0E-9
transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData)
sname_X_loc = 0.0
n = 0
for sname, data_file in some_data:
print("Plotting: "+sname)
station_delay = -data_file.get_nominal_sample_number()*5.0E-9
if sname in guess_delays:
station_delay = guess_delays[sname]
station_delay_points = int(station_delay/5.0E-9)
RFI_filter = RFI_filters[sname]
ant_names = data_file.get_antenna_names()
num_antenna_pairs = int( len( ant_names )/2 )
peak_height = 0.0
print("Block starts at: "+str(block_starts[0]))
for point in block_starts:
T = t0 + point*5.0E-9
for pair in range(num_antenna_pairs):
data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2)
if do_remove_saturation:
remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter( data )
else:
filtered_data = hilbert(data)
even_HE = np.abs(filtered_data)
data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2+1)
if do_remove_saturation:
remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter( data )
else:
filtered_data = hilbert(data)
odd_HE = np.abs(filtered_data)
#ax.plot(T, even_HE + height, 'r')
#ax.plot(T, odd_HE + height, 'g' )
#for cases where data are hard to align due to signal being tiny:
found = False
if len(amplify) > 0:
for key in amplify:
if sname == key:
if omitPOL != 0:
ax.plot(T, amplify[key]*even_HE + height, 'r')
if omitPOL != 1:
ax.plot(T, amplify[key]*odd_HE + height, 'g' )
#ax.plot(T, amplify[key]*even_HE + height, 'r') twinx with block_starts && point as x rather than t! (can I update to latests plot_multiple?)
#ax.plot(T, amplify[key]*odd_HE + height, 'g' )
found = True
if not found:
if omitPOL != 0:
ax.plot(T, even_HE + height, 'r')
if omitPOL != 1:
ax.plot(T, odd_HE + height, 'g' )
#ax2 = ax.twiny()
#xmin = min(block_starts)
#xmax = max(block_starts)
#ax2.set_xlim(xmin, xmax) #can also use ax2.set_xticks(x)
#ax2.xaxis.set_ticks_position('both')
max_even = np.max(even_HE)
if max_even > peak_height:
peak_height = max_even
max_odd = np.max(odd_HE)
if max_odd > peak_height:
peak_height = max_odd
# plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
plt.sca(ax) #somehow this makes it so that the annotations work on all plots except for the first time "Next stations" is clicked
plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform)
ax.ticklabel_format(useOffset=False)
plt.setp(ax.get_xticklabels(), rotation=60, horizontalalignment='right')
plt.subplots_adjust(bottom=0.1,top=0.98,left=0.21)
if ref_plot_peak_location != 0.0:
print("Setting ref peak to " +str(ref_plot_peak_location))
Cpeak = ref_plot_peak_location #peak center, for plotting
Dpeak = 0.0001 # +/- added to peak center for plotting
plt.xlim(Cpeak-Dpeak,Cpeak+Dpeak)
plt.grid(b=True)
height += 2*peak_height
n+=1
def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", max_num_planewaves=np.inf,
positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 50, verbose=True):
left_pulse_length = int(pulse_length/2)
right_pulse_length = pulse_length-left_pulse_length
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = MultiFile_Dal1( raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays )
ant_names = TBB_data.get_antenna_names()
antenna_locations = TBB_data.get_LOFAR_centered_positions()
antenna_delays = TBB_data.get_timing_callibration_delays()
num_antenna_pairs = int( len( ant_names )/2 )
if num_antenna_pairs < min_num_antennas:
return np.array([]), np.array([]), np.array([])
RFI_filter = window_and_filter(timeID=timeID, sname=station)
block_size = RFI_filter.blocksize
out_RMS = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_zenith = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_azimuth = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
N = 0
data = np.empty( (num_antenna_pairs,block_size), dtype=np.double )
for block in range(initial_block, initial_block+number_of_blocks):
if N >= max_num_planewaves:
break
#### open and filter data
for pair_i in range(num_antenna_pairs):
ant_i = pair_i*2 + polarization
data[pair_i,:] = TBB_data.get_data(block*block_size, block_size, antenna_index=ant_i)
remove_saturation(data[pair_i,:], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
np.abs( RFI_filter.filter( data[pair_i,:] ), out=data[pair_i,:])
#### loop over finding planewaves
i = 0
while i < pulses_per_block:
if N >= max_num_planewaves:
break
pulse_location = 0
pulse_amplitude = 0
## find highest peak
for pair_i, HE in enumerate(data):
loc = np.argmax( HE )
amp = HE[ loc ]
if amp > pulse_amplitude:
pulse_amplitude = amp
pulse_location = loc
## check if is strong enough
if pulse_amplitude < min_amplitude:
break
## get
fitter = locatefier()
for pair_i, HE in enumerate(data):
ant_i = pair_i*2 + polarization
signal = np.array( HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] )
if num_double_zeros(signal, threshold=0.1) == 0:
sample_time = 5.0E-9
if upsample_factor > 1:
sample_time /= upsample_factor
signal = resample(signal, len(signal)*upsample_factor )
peak_finder = parabolic_fit( signal )
peak_time = peak_finder.peak_index*sample_time - antenna_delays[ant_i]
fitter.add_antenna(peak_time, antenna_locations[ant_i])
HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] = 0.0
if fitter.num_measurments() < min_num_antennas:
continue
fitter.prep_for_fitting()
X = brute(fitter.RMS, ranges=[[0,np.pi/2],[0,np.pi*2]], Ns=20)
if X[0] >= np.pi/2:
X[0] = (np.pi/2)*0.99
if X[1] >= np.pi*2:
X[1] = (np.pi*2)*0.99
if X[0] < 0:
X[0] = 0.00000001
if X[1] < 0:
X[1] = 0.00000001
ret = least_squares( fitter.time_fits, X, bounds=[[0,0],[np.pi/2,2*np.pi]], ftol=3e-16, xtol=3e-16, gtol=3e-16, x_scale='jac' )
out_RMS[N] = fitter.RMS(ret.x, 3)
out_zenith[N] = ret.x[0]
out_azimuth[N] = ret.x[1]
N += 1
i += 1
return out_RMS[:N], out_zenith[:N], out_azimuth[:N]
