def get_noise_power(TBB_datafile,
RFI_filter,
ant_i,
block_range,
max_fraction_doubleZeros=0.0005):
ret = None
for block in range(block_range[0], block_range[1]):
data = TBB_datafile.get_data(block * RFI_filter.blocksize,
RFI_filter.blocksize,
antenna_index=ant_i)
DBZ_fraction = num_double_zeros(data) / RFI_filter.blocksize
if DBZ_fraction > max_fraction_doubleZeros:
continue
data = RFI_filter.filter(np.array(data, dtype=np.double))
edge_width = int(RFI_filter.blocksize * RFI_filter.half_window_percent)
HE = np.abs(data[edge_width:-edge_width])
noise_power = HE * HE
ave_noise_power = np.average(noise_power)
ret = ave_noise_power
break
return ret
def get_trace_fromIndex(self,
starting_index,
ant_name,
width,
do_remove_RFI=True,
do_remove_saturation=True,
positive_saturation=2046,
negative_saturation=-2047,
removal_length=50,
saturation_half_hann_length=50):
"""similar to previous, but now retrieve trace based on location in file. For repeatability.
Has same returns. predicted arrival time is just the time in middle of trace"""
station_name = SId_to_Sname[int(ant_name[:3])]
station_data = self.data_file_dict[station_name]
data_filter = self.data_filter_dict[station_name]
file_antenna_index = station_data.get_antenna_names().index(ant_name)
total_time_offset = station_data.get_total_delays()[file_antenna_index]
local_data_index = int(data_filter.blocksize * 0.5)
input_data = station_data.get_data(starting_index - local_data_index,
data_filter.blocksize,
antenna_index=file_antenna_index)
input_data = np.array(input_data, dtype=np.double)
if do_remove_saturation:
remove_saturation(input_data, positive_saturation,
negative_saturation, removal_length,
saturation_half_hann_length)
if self.return_dbl_zeros:
dbl_zeros = num_double_zeros(
input_data[local_data_index:local_data_index + width])
if do_remove_RFI:
data = data_filter.filter(
input_data)[local_data_index:local_data_index + width]
else:
data = input_data[local_data_index:local_data_index + width]
predicted_arrival_time = starting_index * 5.0E-9 - total_time_offset + 0.5 * width * 5.0E-9
if self.return_dbl_zeros:
return starting_index, total_time_offset, predicted_arrival_time, data, dbl_zeros
else:
return starting_index, total_time_offset, predicted_arrival_time, data
def FindRFI(TBB_in_file,
block_size,
initial_block,
num_blocks,
max_blocks=None,
verbose=False,
figure_location=None,
lower_frequency=10E6,
upper_frequency=90E6,
num_dbl_z=100):
""" use phase-variance to find RFI in data. TBB_in_file should be a MultiFile_Dal1, encompassing the data for one station. block_size should be around 65536 (2^16).
num_blocks should be at least 20. Sometimes a block needs to be skipped, so max_blocks shows the maximum number of blocks used (after initial block) used to find num_blocks
number of good blocks. initial block should be such that there is no lightning in the max_blocks number of blocks. If max_blocks is None (default), it is set to num_blocks
figure_location should be a folder to save relavent figures, default is None (do not save figures). num_dbl_z is number of double zeros allowed in a block, if there are too
many, then there could be data loss.
returns a dictionary with the following key-value pairs:
"ave_spectrum_magnitude":a numpy array that contains the average of the magnitude of the frequency spectrum
"ave_spectrum_phase": a numpy array containing the average of the phase of the frequency spectrum
"phase_variance":a numpy array containing the phase variance of each frequency channel
"dirty_channels": an array of indeces indicating the channels that are contaminated with RFI
"""
if max_blocks is None:
max_blocks = num_blocks
window_function = half_hann_window(block_size, 0.1)
num_antennas = len(TBB_in_file.get_antenna_names())
if figure_location is not None:
max_over_blocks = np.zeros((num_antennas, max_blocks), dtype=np.double)
#### step one: find which blocks are good, and find average power ####
oneAnt_data = np.empty(block_size, dtype=np.double)
if verbose:
print('finding good blocks')
blocks_good = np.zeros((num_antennas, max_blocks), dtype=bool)
num_good_blocks = np.zeros(num_antennas, dtype=int)
average_power = np.zeros(num_antennas, dtype=np.double)
for block_i in range(max_blocks):
block = block_i + initial_block
for ant_i in range(num_antennas):
oneAnt_data[:] = TBB_in_file.get_data(block_size * block,
block_size,
antenna_index=ant_i)
if num_double_zeros(
oneAnt_data
) < num_dbl_z: ## this antenna on this block is good
blocks_good[ant_i, block_i] = True
num_good_blocks[ant_i] += 1
oneAnt_data *= window_function
FFT_data = np.fft.fft(oneAnt_data)
np.abs(FFT_data, out=FFT_data)
magnitude = FFT_data
magnitude *= magnitude
average_power[ant_i] += np.real(np.sum(magnitude))
# else:
# print(block_i, ant_i, num_double_zeros( oneAnt_data ))
# plt.plot(oneAnt_data)
# plt.show()
if figure_location is not None:
max_over_blocks[ant_i, block_i] = np.max(oneAnt_data[ant_i])
average_power[num_good_blocks != 0] /= num_good_blocks[
num_good_blocks != 0]
#### now we try to find the best referance antenna, Require that antenan allows for maximum number of good antennas, and has **best** average recieved power
allowed_num_antennas = np.empty(
num_antennas, dtype=np.int
) ## if ant_i is choosen to be your referance antnena, then allowed_num_antennas[ ant_i ] is the number of antennas with num_blocks good blocks
for ant_i in range(num_antennas): ### fill allowed_num_antennas
blocks_can_use = np.where(blocks_good[ant_i])[0]
num_good_blocks_per_antenna = np.sum(blocks_good[:, blocks_can_use],
axis=1)
allowed_num_antennas[ant_i] = np.sum(
num_good_blocks_per_antenna >= num_blocks)
max_allowed_antennas = np.max(allowed_num_antennas)
if max_allowed_antennas < 2:
print("ERROR: station", TBB_in_file.get_station_name(),
"cannot find RFI")
return
## pick ref antenna that allows max number of atnennas, and has most median amount of power
can_be_ref_antenna = (allowed_num_antennas == max_allowed_antennas)
sorted_by_power = np.argsort(average_power)
mps = median_sorted_by_power(sorted_by_power)
for ant_i in mps:
if can_be_ref_antenna[ant_i]:
ref_antenna = ant_i
break
if verbose:
print('Taking channel %d as reference antenna' % ref_antenna)
## define some helping variables ##
good_blocks = np.where(blocks_good[ref_antenna])[0]
num_good_blocks = np.sum(blocks_good[:, good_blocks], axis=1)
antenna_is_good = num_good_blocks >= num_blocks
blocks_good[np.logical_not(antenna_is_good), :] = False
#### process data ####
num_processed_blocks = np.zeros(num_antennas, dtype=int)
frequencies = np.fft.fftfreq(block_size,
1.0 / TBB_in_file.get_sample_frequency())
lower_frequency_index = np.searchsorted(
frequencies[:int(len(frequencies) / 2)], lower_frequency)
upper_frequency_index = np.searchsorted(
frequencies[:int(len(frequencies) / 2)], upper_frequency)
phase_mean = np.zeros(
(num_antennas, upper_frequency_index - lower_frequency_index),
dtype=complex)
spectrum_mean = np.zeros(
(num_antennas, upper_frequency_index - lower_frequency_index),
dtype=np.double)
data = np.empty((num_antennas, len(frequencies)), dtype=np.complex)
temp_mag_spectrum = np.empty((num_antennas, len(frequencies)),
dtype=np.double)
temp_phase_spectrum = np.empty((num_antennas, len(frequencies)),
dtype=np.complex)
for block_i in good_blocks:
block = block_i + initial_block
if verbose:
print('Doing block %d' % block)
for ant_i in range(num_antennas):
if (num_processed_blocks[ant_i] == num_blocks
and not ant_i == ref_antenna) or not blocks_good[ant_i,
block_i]:
continue
oneAnt_data[:] = TBB_in_file.get_data(block_size * block,
block_size,
antenna_index=ant_i)
##window the data
# Note: No hanning window if we want to measure power accurately from spectrum
# in the same units as power from timeseries. Applying a window gives (at least) a scale factor
# difference!
# But no window makes the cleaning less effective... :(
oneAnt_data *= window_function
data[ant_i] = np.fft.fft(oneAnt_data)
data /= block_size
np.abs(data, out=temp_mag_spectrum)
temp_phase_spectrum[:] = data
temp_phase_spectrum /= (temp_mag_spectrum + 1.0E-15)
temp_phase_spectrum[:, :] /= temp_phase_spectrum[ref_antenna, :]
temp_mag_spectrum *= temp_mag_spectrum ## square
for ant_i in range(num_antennas):
if (num_processed_blocks[ant_i] == num_blocks
and not ant_i == ref_antenna) or not blocks_good[ant_i,
block_i]:
continue
phase_mean[ant_i, :] += temp_phase_spectrum[ant_i][
lower_frequency_index:upper_frequency_index]
spectrum_mean[ant_i, :] += temp_mag_spectrum[ant_i][
lower_frequency_index:upper_frequency_index]
num_processed_blocks[ant_i] += 1
if np.min(num_processed_blocks[antenna_is_good]) == num_blocks:
break
if verbose:
print(num_blocks, "analyzed blocks", np.sum(antenna_is_good),
"analyzed antennas out of", len(antenna_is_good))
## get only good antennas
antenna_is_good[
ref_antenna] = False ## we don't want to analyze the phase stability of the referance antenna
### get mean and phase stability ###
spectrum_mean /= num_blocks
phase_stability = np.abs(phase_mean)
phase_stability *= -1.0 / num_blocks
phase_stability += 1.0
#### get median of stability by channel, across each antenna ###
median_phase_spread_byChannel = np.median(phase_stability[antenna_is_good],
axis=0)
#### get median across all chanells
median_spread = np.median(median_phase_spread_byChannel)
#### create a noise cuttoff###
sorted_phase_spreads = np.sort(median_phase_spread_byChannel)
N = len(median_phase_spread_byChannel)
noise = sorted_phase_spreads[int(N * 0.95)] - sorted_phase_spreads[int(
N / 2)]
#### get channels contaminated by RFI, where phase stability is smaller than noise ###
dirty_channels = np.where(
median_phase_spread_byChannel < (median_spread - 3 * noise))[0]
### extend dirty channels by some size, in order to account for shoulders ####
extend_dirty_channels = np.zeros(N, dtype=bool)
half_flagwidth = int(block_size / 8192)
for i in dirty_channels:
flag_min = i - half_flagwidth
flag_max = i + half_flagwidth
if flag_min < 0:
flag_min = 0
if flag_max >= N:
flag_max = N - 1
extend_dirty_channels[flag_min:flag_max] = True
dirty_channels = np.where(extend_dirty_channels)[0]
antenna_is_good[ref_antenna] = True ## cause'.... ya know.... it is
#### plot and return data ####
frequencies = frequencies[lower_frequency_index:upper_frequency_index]
if figure_location is not None:
frequencies_MHZ = frequencies * 1.0E-6
plt.figure()
plt.plot(frequencies_MHZ, median_phase_spread_byChannel)
plt.axhline(median_spread - 3 * noise, color='r')
plt.title(
"Phase spread vs frequency. Red horizontal line shows cuttoff.")
plt.ylabel("Spread value")
plt.xlabel("Frequency [MHz]")
# plt.legend()
if figure_location == "show":
plt.show()
else:
plt.savefig(figure_location + '/phase_spreads.png')
plt.close()
plt.figure()
plt.plot(frequencies_MHZ, spectrum_mean[ref_antenna])
plt.plot(frequencies_MHZ[dirty_channels],
spectrum_mean[ref_antenna][dirty_channels], 'ro')
plt.xlabel("Frequency [MHz]")
plt.ylabel("magnitude")
plt.yscale('log', nonposy='clip')
# plt.legend()
if figure_location == "show":
plt.show()
else:
plt.savefig(figure_location + '/magnitude.png')
plt.close()
plt.figure()
for maxes, ant_name in zip(max_over_blocks,
TBB_in_file.get_antenna_names()):
plt.plot(maxes, label=ant_name)
plt.ylabel("maximum")
plt.xlabel("block index")
plt.legend()
if figure_location == "show":
plt.show()
else:
plt.savefig(figure_location + '/max_over_blocks.png')
plt.close()
output_dict = {}
output_dict["ave_spectrum_magnitude"] = spectrum_mean
output_dict["ave_spectrum_phase"] = np.angle(phase_mean, deg=False)
output_dict["phase_variance"] = phase_stability
output_dict["dirty_channels"] = dirty_channels + lower_frequency_index
output_dict["blocksize"] = block_size
cleaned_spectrum = np.array(spectrum_mean)
cleaned_spectrum[:, dirty_channels] = 0.0
output_dict["cleaned_spectrum_magnitude"] = cleaned_spectrum
output_dict["cleaned_power"] = 2 * np.sum(cleaned_spectrum, axis=1)
print('cleaned power:', output_dict["cleaned_power"])
print('old power', 2 * np.sum(spectrum_mean, axis=1))
print('num dirty channels:', len(dirty_channels))
output_dict["antenna_names"] = TBB_in_file.get_antenna_names()
output_dict["timestamp"] = TBB_in_file.get_timestamp()
output_dict["antennas_good"] = antenna_is_good
output_dict["frequency"] = frequencies
return output_dict
def process_block(self, start_index, block_index, h5_groupobject, log_func=do_nothing):
#### open and filter the data ####
prefered_ant_i = None
prefered_ant_dataLoss = np.inf
for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use):
data_file = self.input_files[station_i]
RFI_filter = self.RFI_filters[station_i]
offset = self.antenna_data_offsets[ant_i]
self.data_block[ant_i, :] = data_file.get_data(start_index+offset, self.block_size, antenna_index=station_ant_i) ## get the data. accounting for the offsets calculated earlier
remove_saturation(self.data_block[ant_i, :], self.positive_saturation, self.negative_saturation, post_removal_length=self.saturation_removal_length,
half_hann_length=self.saturation_hann_window_length)
if ant_i in self.station_to_antenna_indeces_dict[ self.prefered_station ]: ## this must be done before filtering
num_D_zeros = num_double_zeros( self.data_block[ant_i, :] )
if num_D_zeros < prefered_ant_dataLoss: ## this antenna could be teh antenna, in the prefered station, with least data loss
prefered_ant_dataLoss = num_D_zeros
prefered_ant_i = ant_i
self.data_block[ant_i, :] = RFI_filter.filter( self.data_block[ant_i, :] )
### make output object ###
h5_groupobject = h5_groupobject.create_group(str(block_index))
h5_groupobject.attrs['block_index'] = block_index
h5_groupobject.attrs['start_index'] = start_index
h5_groupobject.attrs['prefered_ant_i'] = prefered_ant_i
#### find sources ####
np.abs( self.data_block[ prefered_ant_i ], out=self.hilbert_envelope_tmp )
for event_i in range(self.max_events_perBlock):
## find peak ##
peak_loc = np.argmax(self.hilbert_envelope_tmp[self.startBlock_exclusion : -self.endBlock_exclusion ]) + self.startBlock_exclusion
trace_start_loc = peak_loc - int( self.pulse_length/2 ) # pobably account for fact that trace is now longer than pulse_length on prefered antenna
if self.hilbert_envelope_tmp[peak_loc] < self.min_pref_ant_amplitude:
log_func("peaks are too small. Done searching")
break
## select data for stage one ##
s1_ant_i = 0
for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use):
if self.use_core_stations_S1 or self.is_not_core[ant_i]:
half_window_length = self.half_antenna_data_length[ant_i]
self.stage_1_imager.set_data( self.data_block[ant_i][trace_start_loc : trace_start_loc+2*half_window_length], s1_ant_i )
s1_ant_i += 1
## fft and xcorrelation ##
self.stage_1_imager.prepare_image( prefered_ant_i, self.min_pulse_amplitude )
log_func("source:", event_i)
stage_1_result, num_itter, num_stage1_itters = stochastic_minimizer(self.stage_1_imager.intensity, self.bounding_box, converg_num=self.stage_1_converg_num,
converg_rad=self.stage_1_converg_radius, max_itters=self.stage_1_max_itters)
log_func(" stoch. itters:", num_itter, num_stage1_itters)
## select data for stage 2 ##
previous_solution = stage_1_result
converged = False
for stage2loop_i in range(self.stage_2_max_itters):
problem = False
s2_ant_i = -1
for ant_i in range( self.num_antennas ):
if self.use_core_stations_S2 or self.is_not_core[ant_i]:
s2_ant_i += 1 ## do this here cause of break below
modeled_dt = -( np.linalg.norm( self.antenna_locations[ prefered_ant_i ]-previous_solution.x ) -
np.linalg.norm( self.antenna_locations[ant_i]-previous_solution.x ) )/v_air
modeled_dt -= self.antenna_delays[ prefered_ant_i ] - self.antenna_delays[ant_i]
modeled_dt /= 5.0E-9
modeled_dt += peak_loc
modeled_dt = int(modeled_dt)
if modeled_dt+int(self.pulse_length/2) >= len(self.data_block[ant_i]):
problem = True
break
self.stage_2_imager.set_data( self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)]*self.stage_2_window, s2_ant_i, -modeled_dt*5.0E-9 )
if problem:
log_func("unknown problem. LOC at", previous_solution.x)
self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] = 0.0
continue
## fft and and xcorrelation ##
self.stage_2_imager.prepare_image( self.min_pulse_amplitude )
#
BB = np.array( [ [previous_solution.x[0]-50, previous_solution.x[0]+50], [previous_solution.x[1]-50, previous_solution.x[1]+50], [previous_solution.x[2]-50, previous_solution.x[2]+50] ] )
stage_2_result, s2_itternum, num_stage2_itters = stochastic_minimizer(self.stage_2_imager.intensity_ABSbefore , BB, converg_num=5, test_spot=previous_solution.x,
converg_rad=self.stage_2_convergence_length, max_itters=self.stage_2_max_stoch_itters, options={'maxiter':1000})
D = np.linalg.norm( stage_2_result.x - previous_solution.x )
log_func(" s2 itter: {:2d} {:4.2f} {:d}".format(stage2loop_i, -stage_2_result.fun, int(D)) )
if D < self.stage_2_convergence_length:
converged = True
break
elif D > self.stage_2_break_length:
converged = False
break
previous_solution = stage_2_result
new_stage_1_result = minimize(self.stage_1_imager.intensity, stage_2_result.x, method="Nelder-Mead", options={'maxiter':1000})
log_func(" old S1: {:4.2f} new SH1: {:4.2f}".format(-stage_1_result.fun, -new_stage_1_result.fun) )
if stage_1_result.fun < new_stage_1_result.fun:
S1_S2_distance = np.linalg.norm(stage_1_result.x-stage_2_result.x)
else:
S1_S2_distance = np.linalg.norm(new_stage_1_result.x-stage_2_result.x)
log_func(" loc: {:d} {:d} {:d}".format(int(stage_2_result.x[0]), int(stage_2_result.x[1]), int(stage_2_result.x[2])) )
log_func(" S1-S2 distance: {:d} converged: {} ".format( int(S1_S2_distance), converged) )
log_func(" intensity: {:4.2f} amplitude: {:d} ".format( -stage_2_result.fun, int(self.hilbert_envelope_tmp[peak_loc])) )
log_func()
log_func()
## save to file ##
source_dataset = h5_groupobject.create_dataset(str(event_i), (self.num_antennas,self.pulse_length), dtype=np.complex)
source_dataset.attrs["loc"] = stage_2_result.x
source_dataset.attrs["unique_index"] = block_index*self.max_events_perBlock + event_i
source_time_s2 = (peak_loc+start_index+self.antenna_data_offsets[prefered_ant_i])*5.0E-9 - np.linalg.norm( stage_2_result.x - self.antenna_locations[ prefered_ant_i ] )/v_air
source_time_s2 -= self.prefered_station_timing_offset + self.prefered_station_antenna_timing_offsets[ self.antennas_to_use[prefered_ant_i][1] ]
source_dataset.attrs["T"] = source_time_s2
source_dataset.attrs["peak_index"] = peak_loc
source_dataset.attrs["intensity"] = -stage_2_result.fun
source_dataset.attrs["stage_1_success"] = (num_stage1_itters==self.stage_1_converg_num)
source_dataset.attrs["stage_1_num_itters"] = num_stage1_itters
source_dataset.attrs["amplitude"] = self.hilbert_envelope_tmp[peak_loc]
source_dataset.attrs["S1_S2_distance"] = S1_S2_distance
source_dataset.attrs["converged"] = converged
source_time_s1 = (peak_loc+start_index+self.antenna_data_offsets[prefered_ant_i])*5.0E-9 - np.linalg.norm( stage_1_result.x - self.antenna_locations[ prefered_ant_i ] )/v_air
source_time_s1 -= self.prefered_station_timing_offset + self.prefered_station_antenna_timing_offsets[ self.antennas_to_use[prefered_ant_i][1] ]
source_dataset.attrs["XYZT_s1"] = np.append(stage_1_result.x, [source_time_s1])
#### erase the peaks !! ####
# self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] *= self.erasure_window
self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] = 0.0
if converged and self.erase_pulses:
for ant_i in range( self.num_antennas ):
modeled_dt = -( np.linalg.norm( self.antenna_locations[ prefered_ant_i ]-stage_2_result.x ) -
np.linalg.norm( self.antenna_locations[ant_i]-stage_2_result.x ) )/v_air
modeled_dt -= self.antenna_delays[ prefered_ant_i ] - self.antenna_delays[ant_i]
modeled_dt /= 5.0E-9
modeled_dt += peak_loc
modeled_dt = int(modeled_dt)
source_dataset[ant_i] = self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)]
self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)] *= self.erasure_window
def get_noise_std(timeID,
initial_block,
max_num_blocks,
max_double_zeros=100,
stations=None,
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt"):
processed_data_folder = processed_data_dir(timeID)
if isinstance(polarization_flips, str):
polarization_flips = read_antenna_pol_flips(processed_data_folder +
'/' + polarization_flips)
if isinstance(processed_data_folder, str):
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
half_window_percent = 0.1
half_window_percent *= 1.1 ## just to be sure we are away from edge
raw_fpaths = filePaths_by_stationName(timeID)
if stations is None:
stations = raw_fpaths.keys()
out_dict = {}
for sname in stations:
print(sname)
TBB_data = MultiFile_Dal1(raw_fpaths[sname],
polarization_flips=polarization_flips,
bad_antennas=bad_antennas)
RFI_filter = window_and_filter(timeID=timeID, sname=sname)
block_size = RFI_filter.blocksize
edge_size = int(half_window_percent * block_size)
antenna_names = TBB_data.get_antenna_names()
measured_std = np.zeros(len(antenna_names))
measured_std[:] = -1
for block_i in range(initial_block, initial_block + max_num_blocks):
for ant_i in range(len(antenna_names)):
if measured_std[ant_i] > 0:
continue ## we got a measurment, so skip it
data = np.array(TBB_data.get_data(block_i * block_size,
block_size,
antenna_index=ant_i),
dtype=np.double)
if num_double_zeros(data) > max_double_zeros:
continue ## bad block, skip
filtered_data = np.real(RFI_filter.filter(data))
filtered_data = filtered_data[edge_size:
-edge_size] ##avoid the edge
measured_std[ant_i] = np.std(filtered_data)
if np.all(measured_std > 0): ## we can break early
break
for ant_name, std in zip(antenna_names, measured_std):
out_dict[ant_name] = std
filter = measured_std > 0
print(" ave std:", np.average(measured_std[filter]))
print(" total ant:", len(filter), "good ant:", np.sum(filter))
print()
return out_dict
def __init__(self,
TBB_data,
polarization,
initial_block,
number_of_blocks,
pulses_per_block=10,
pulse_length=50,
min_amplitude=50,
upsample_factor=0,
min_num_antennas=4,
max_num_planewaves=np.inf,
timeID=None,
blocksize=2**16,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
verbose=True):
self.parabolic_fitter = parabolic_fitter()
self.polarization = polarization
self.TBB_data = TBB_data
left_pulse_length = int(pulse_length / 2)
right_pulse_length = pulse_length - left_pulse_length
ant_names = TBB_data.get_antenna_names()
num_antenna_pairs = int(len(ant_names) / 2)
self.num_found_planewaves = 0
if num_antenna_pairs < min_num_antennas:
return
RFI_filter = window_and_filter(
timeID=timeID,
sname=TBB_data.get_station_name(),
blocksize=(blocksize if timeID is None else None))
block_size = RFI_filter.blocksize
self.num_found_planewaves = 0
data = np.empty((num_antenna_pairs, block_size), dtype=np.double)
self.planewave_data = []
self.planewave_second_derivatives = []
for block in range(initial_block, initial_block + number_of_blocks):
if self.num_found_planewaves >= max_num_planewaves:
break
#### open and filter data
# print('open block', block)
for pair_i in range(num_antenna_pairs):
ant_i = pair_i * 2 + polarization
data[pair_i, :] = TBB_data.get_data(block * block_size,
block_size,
antenna_index=ant_i)
remove_saturation(data[pair_i, :], positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
np.abs(RFI_filter.filter(data[pair_i, :]), out=data[pair_i, :])
#### loop over finding planewaves
i = 0
while i < pulses_per_block:
if self.num_found_planewaves >= max_num_planewaves:
break
pulse_location = 0
pulse_amplitude = 0
## find highest peak
for pair_i, HE in enumerate(data):
loc = np.argmax(HE)
amp = HE[loc]
if amp > pulse_amplitude:
pulse_amplitude = amp
pulse_location = loc
## check if is strong enough
if pulse_amplitude < min_amplitude:
break
## get
newPW_data = np.full(num_antenna_pairs,
np.nan,
dtype=np.double)
newPW_derivatives = np.full(num_antenna_pairs,
np.nan,
dtype=np.double)
for pair_i, HE in enumerate(data):
# ant_i = pair_i*2 + polarization
signal = np.array(
HE[pulse_location - left_pulse_length:pulse_location +
right_pulse_length])
if num_double_zeros(signal, threshold=0.1) == 0:
sample_time = 5.0E-9
if upsample_factor > 1:
sample_time /= upsample_factor
signal = resample(signal,
len(signal) * upsample_factor)
newPW_data[pair_i] = self.parabolic_fitter.fit(
signal) * sample_time
newPW_derivatives[
pair_i] = self.parabolic_fitter.second_derivative(
) * sample_time
HE[pulse_location - left_pulse_length:pulse_location +
right_pulse_length] = 0.0
if np.sum(np.isfinite(newPW_data)) >= min_num_antennas:
self.planewave_data.append(newPW_data)
self.planewave_second_derivatives.append(newPW_derivatives)
self.num_found_planewaves += 1
i += 1
processed_data_dir = processed_data_dir(timeID)
output_fpath = processed_data_dir + output_folder
if not isdir(output_fpath):
mkdir(output_fpath)
log.set(output_fpath + '/log.txt', True)
log.take_stdout()
for station, fpaths in raw_fpaths.items():
print(station)
TBB_data = MultiFile_Dal1(fpaths)
for i, ant_name in enumerate(TBB_data.get_antenna_names()):
num_data_points = 0
num_zeros = 0
for block in range(initial_block, final_block):
data = TBB_data.get_data(block * block_size,
block_size,
antenna_index=i)
num_zeros += num_double_zeros(data)
num_data_points += block_size
print(i, ant_name, 100.0 * float(num_zeros) / num_data_points)
print()
def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", max_num_planewaves=np.inf,
positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 50, verbose=True):
left_pulse_length = int(pulse_length/2)
right_pulse_length = pulse_length-left_pulse_length
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = MultiFile_Dal1( raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays )
ant_names = TBB_data.get_antenna_names()
antenna_locations = TBB_data.get_LOFAR_centered_positions()
antenna_delays = TBB_data.get_timing_callibration_delays()
num_antenna_pairs = int( len( ant_names )/2 )
if num_antenna_pairs < min_num_antennas:
return np.array([]), np.array([]), np.array([])
RFI_filter = window_and_filter(timeID=timeID, sname=station)
block_size = RFI_filter.blocksize
out_RMS = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_zenith = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_azimuth = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
N = 0
data = np.empty( (num_antenna_pairs,block_size), dtype=np.double )
for block in range(initial_block, initial_block+number_of_blocks):
if N >= max_num_planewaves:
break
#### open and filter data
for pair_i in range(num_antenna_pairs):
ant_i = pair_i*2 + polarization
data[pair_i,:] = TBB_data.get_data(block*block_size, block_size, antenna_index=ant_i)
remove_saturation(data[pair_i,:], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
np.abs( RFI_filter.filter( data[pair_i,:] ), out=data[pair_i,:])
#### loop over finding planewaves
i = 0
while i < pulses_per_block:
if N >= max_num_planewaves:
break
pulse_location = 0
pulse_amplitude = 0
## find highest peak
for pair_i, HE in enumerate(data):
loc = np.argmax( HE )
amp = HE[ loc ]
if amp > pulse_amplitude:
pulse_amplitude = amp
pulse_location = loc
## check if is strong enough
if pulse_amplitude < min_amplitude:
break
## get
fitter = locatefier()
for pair_i, HE in enumerate(data):
ant_i = pair_i*2 + polarization
signal = np.array( HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] )
if num_double_zeros(signal, threshold=0.1) == 0:
sample_time = 5.0E-9
if upsample_factor > 1:
sample_time /= upsample_factor
signal = resample(signal, len(signal)*upsample_factor )
peak_finder = parabolic_fit( signal )
peak_time = peak_finder.peak_index*sample_time - antenna_delays[ant_i]
fitter.add_antenna(peak_time, antenna_locations[ant_i])
HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] = 0.0
if fitter.num_measurments() < min_num_antennas:
continue
fitter.prep_for_fitting()
X = brute(fitter.RMS, ranges=[[0,np.pi/2],[0,np.pi*2]], Ns=20)
if X[0] >= np.pi/2:
X[0] = (np.pi/2)*0.99
if X[1] >= np.pi*2:
X[1] = (np.pi*2)*0.99
if X[0] < 0:
X[0] = 0.00000001
if X[1] < 0:
X[1] = 0.00000001
ret = least_squares( fitter.time_fits, X, bounds=[[0,0],[np.pi/2,2*np.pi]], ftol=3e-16, xtol=3e-16, gtol=3e-16, x_scale='jac' )
out_RMS[N] = fitter.RMS(ret.x, 3)
out_zenith[N] = ret.x[0]
out_azimuth[N] = ret.x[1]
N += 1
i += 1
return out_RMS[:N], out_zenith[:N], out_azimuth[:N]
def get_trace_fromLoc(self,
XYZT,
ant_name,
width,
do_remove_RFI=True,
do_remove_saturation=True,
positive_saturation=2046,
negative_saturation=-2047,
removal_length=50,
half_hann_length=50):
"""given the location of the source in XYZT, name of the antenna, and width (in num data samples) of the desired pulse,
return starting_index of the returned trace, total time calibration delay of that antenna (from TBB.get_total_delays), predicted arrival time of pulse at that antenna,
and the time trace centered on the arrival time."""
station_name = SId_to_Sname[int(ant_name[:3])]
station_data = self.data_file_dict[station_name]
data_filter = self.data_filter_dict[station_name]
file_antenna_index = station_data.get_antenna_names().index(ant_name)
# ant_loc = station_data.get_LOFAR_centered_positions()[ file_antenna_index ]
# ant_time_offset = station_data.get_timing_callibration_delays()[file_antenna_index] - station_data.get_nominal_sample_number()*5.0E-9
# total_time_offset = ant_time_offset + self.station_timing_calibration[station_name]
# predicted_arrival_time = np.linalg.norm( XYZT[:3]-ant_loc )/v_air + XYZT[3]
total_time_offset = station_data.get_total_delays()[file_antenna_index]
antenna_locations = station_data.get_LOFAR_centered_positions()
predicted_arrival_time = station_data.get_geometric_delays(
XYZT[:3],
antenna_locations=antenna_locations[
file_antenna_index:file_antenna_index + 1]) + XYZT[3]
data_arrival_index = int(predicted_arrival_time / 5.0E-9 +
total_time_offset / 5.0E-9)
local_data_index = int(data_filter.blocksize * 0.5)
data_start_sample = data_arrival_index - local_data_index
input_data = station_data.get_data(data_start_sample,
data_filter.blocksize,
antenna_index=file_antenna_index)
input_data = np.array(input_data, dtype=np.double)
if do_remove_saturation:
remove_saturation(input_data, positive_saturation,
negative_saturation, removal_length,
half_hann_length)
width_before = int(width * 0.5)
width_after = width - width_before
if self.return_dbl_zeros:
dbl_zeros = num_double_zeros(
input_data[local_data_index - width_before:local_data_index +
width_after])
if do_remove_RFI:
data = data_filter.filter(
input_data)[local_data_index - width_before:local_data_index +
width_after]
else:
data = input_data[local_data_index -
width_before:local_data_index + width_after]
if self.return_dbl_zeros:
return data_start_sample + local_data_index - width_before, total_time_offset, predicted_arrival_time, data, dbl_zeros
else:
return data_start_sample + local_data_index - width_before, total_time_offset, predicted_arrival_time, data
def FindEnergy_BothPolarizations(self,
source_bounds,
min_amp=None,
ret_amps=False,
plot=False):
edge = self.edge_width + 20 # a little extra for pulse width
usable_blockwidth = self.blocksize - 2 * edge
bounds_corners = corners_of_bounds(source_bounds)
largest_edge = 0
for C1 in bounds_corners:
for C2 in bounds_corners:
L = np.linalg.norm(C2 - C1)
# print(L, C1, C2)
if L > largest_edge:
largest_edge = L
new_XYZT_bounds = np.array(source_bounds)
new_XYZT_bounds[3, 0] -= largest_edge / v_air
new_XYZT_bounds[3, 1] += largest_edge / v_air
sources, indeces_by_source = self.sources_in_bounds(new_XYZT_bounds)
sources = list(sources)
indeces_by_antenna = [[] for i in range(self.num_antennas)]
for source_ilist in indeces_by_source:
for anti, i in enumerate(source_ilist):
indeces_by_antenna[anti].append(i)
# source_XYZTs_in_bounds = np.array( [source.XYZT for source in sources ] )
locX = (source_bounds[0, 0] + source_bounds[0, 1]) / 2
locY = (source_bounds[1, 0] + source_bounds[1, 1]) / 2
locZ = (source_bounds[2, 0] + source_bounds[2, 1]) / 2
XYZT_start = np.array([locX, locY, locZ, source_bounds[3, 0]])
samples_left = int(
(source_bounds[3, 1] - source_bounds[3, 0]) / 5.0e-9)
current_sample = np.array([
self.TraceLocator.source_recieved_index(XYZT_start, ant_name)
for ant_name in self.antenna_names
])
# event_samples = [ np.sort([self.TraceLocator.source_recieved_index(XYZT, ant_name ) for XYZT in source_XYZTs_in_bounds]) for ant_name in self.antenna_names ]
event_samples = [np.sort(I) for I in indeces_by_antenna]
current_event_index = [0] * self.num_antennas
print('num total source:', len(sources))
total_energy = 0.0
imaged_energy = 0.0
amps = []
while samples_left > 0:
width = usable_blockwidth
if samples_left < width:
width = samples_left
samples_left -= width
### FIRST FOR EVENS
even_ant_found = False
for ant_pair_i in range(self.num_pairs):
ant_i = ant_pair_i * 2
noise_power = self.antenna_noise_power[ant_i]
if noise_power is None:
continue
start_sample = current_sample[ant_i]
data = self.TBB_datafile.get_data(start_sample - edge,
self.RFI_filter.blocksize,
antenna_index=ant_i)
DBZ_fraction = num_double_zeros(
data) / self.RFI_filter.blocksize
if DBZ_fraction > self.max_fraction_doubleZeros:
continue
even_ant_found = True
data = self.RFI_filter.filter(np.array(data, dtype=np.double))
HE = np.abs(data)
if plot:
T = np.arange(
self.RFI_filter.blocksize) + (start_sample - edge)
plt.plot(T, HE)
power = HE * HE
N = noise_power * width * 5.0E-9
if min_amp is None:
power_to_integrate = power
else:
power_to_integrate = np.array(power)
power_to_integrate[HE < min_amp] = 0.0
# print('ARG:', np.sum(HE<min_amp))
N *= 0.0000000001
# recieved energy
A = simps(power_to_integrate[edge:edge + width], dx=5.0E-9)
energy = A - N
if energy < 0:
energy = 0.001 * N
total_energy += energy
# imaged energy
N = 0
# used_samples = []
event_sample_numbers = event_samples[ant_i]
last_sample_number = 0
for source_i in range(current_event_index[ant_i],
len(event_sample_numbers)):
source_sample_number = event_sample_numbers[source_i]
if source_sample_number < start_sample:
last_sample_number = source_i
elif start_sample <= source_sample_number < start_sample + width:
# used_samples.append( source_sample_number )
last_sample_number = source_i
local_sample_number = source_sample_number - (
start_sample - edge)
A = HE[local_sample_number - 3:local_sample_number + 3]
E = np.max(A)
if E >= min_amp:
imaged_energy += E * E
N += 1
if ret_amps:
amps.append(E)
else:
break
current_event_index[ant_i] = last_sample_number
# print('E:', ant_i, 'samples:', start_sample, start_sample+width)
# print(' NS:', N)
# print(' used sources locs', used_samples)
# if N > 0:
# print(' final source loc:', event_sample_numbers[last_sample_number] )
# print( event_sample_numbers )
break
if not even_ant_found:
print("ERROR! too much data loss on even antenna!")
quit()
### Odd antennas
odd_ant_found = False
for ant_pair_i in range(self.num_pairs):
ant_i = ant_pair_i * 2 + 1
noise_power = self.antenna_noise_power[ant_i]
if noise_power is None:
continue
start_sample = current_sample[ant_i]
data = self.TBB_datafile.get_data(start_sample - edge,
self.RFI_filter.blocksize,
antenna_index=ant_i)
DBZ_fraction = num_double_zeros(
data) / self.RFI_filter.blocksize
if DBZ_fraction > self.max_fraction_doubleZeros:
continue
odd_ant_found = True
data = self.RFI_filter.filter(np.array(data, dtype=np.double))
HE = np.abs(data)
if plot:
T = np.arange(
self.RFI_filter.blocksize) + (start_sample - edge)
plt.plot(T, HE)
# print("odd num GE:", np.sum())
power = HE * HE
N = noise_power * width * 5.0E-9
if min_amp is None:
power_to_integrate = power
else:
power_to_integrate = np.array(power)
power_to_integrate[HE < min_amp] = 0.0
# print('ARG:', np.sum(HE<min_amp))
N *= 0.0000000001
# recieved energy
A = simps(power_to_integrate[edge:edge + width], dx=5.0E-9)
energy = A - N
if energy < 0:
energy = 0.001 * N
total_energy += energy
# imaged energy
N = 0
event_sample_numbers = event_samples[ant_i]
last_sample_number = 0
for source_i in range(current_event_index[ant_i],
len(event_sample_numbers)):
source_sample_number = event_sample_numbers[source_i]
if source_sample_number < start_sample:
last_sample_number = source_i
elif start_sample <= source_sample_number < start_sample + width:
last_sample_number = source_i
local_sample_number = source_sample_number - (
start_sample - edge)
A = HE[local_sample_number - 3:local_sample_number + 3]
E = np.max(A)
if E >= min_amp:
imaged_energy += E * E
N += 1
# if ret_amps :
# amps.append(E)
else:
break
current_event_index[ant_i] = last_sample_number
break
if not odd_ant_found:
print("ERROR! too much data loss on even antenna!")
quit()
## update
current_sample += width
if plot:
print(total_energy, imaged_energy * self.pulse_width)
plt.show()
if ret_amps:
return total_energy, imaged_energy * self.pulse_width, amps
else:
return total_energy, imaged_energy * self.pulse_width
