def planewave_fits(timeID,
station,
polarization,
initial_block,
number_of_blocks,
pulses_per_block=10,
pulse_length=50,
min_amplitude=50,
upsample_factor=0,
min_num_antennas=4,
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt",
max_num_planewaves=np.inf,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
verbose=True):
""" wrapper around 'planewave_fitter' only present for backwards compatiblility. Returns RMS, zenithal and azimuthal fit values, as three arrays in that order"""
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder +
'/' +
additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = MultiFile_Dal1(raw_fpaths[station],
polarization_flips=polarization_flips,
bad_antennas=bad_antennas,
additional_ant_delays=additional_antenna_delays)
fitter = planewave_fitter(
TBB_data=TBB_data,
polarization=polarization,
initial_block=initial_block,
number_of_blocks=number_of_blocks,
pulses_per_block=pulses_per_block,
pulse_length=pulse_length,
min_amplitude=min_amplitude,
upsample_factor=upsample_factor,
min_num_antennas=min_num_antennas,
max_num_planewaves=max_num_planewaves,
verbose=verbose, ## doesn't do anything anyway
positive_saturation=positive_saturation,
negative_saturation=negative_saturation,
saturation_post_removal_length=saturation_post_removal_length,
saturation_half_hann_length=saturation_half_hann_length)
RMSs, Zeniths, Azimuths, throw = fitter.go_fit()
return RMSs, Zeniths, Azimuths
from stokesIO import read_polarization_data, save_acceleration_vector
timeID = "D20190424T210306.154Z"
utilities.default_raw_data_loc = "/home/student4/Marten/KAP_data_link/lightning_data"
utilities.default_processed_data_loc = "/home/student4/Marten/processed_files"
processed_data_folder = processed_data_dir(timeID)
station_delay_file = "station_delays.txt"
polarization_flips = "polarization_flips.txt"
bad_antennas = "bad_antennas.txt"
additional_antenna_delays = "ant_delays.txt"
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' +
additional_antenna_delays)
station_timing_offsets = read_station_delays(processed_data_folder + '/' +
station_delay_file)
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = {
sname: MultiFile_Dal1(fpath,
force_metadata_ant_pos=True,
polarization_flips=polarization_flips,
bad_antennas=bad_antennas,
additional_ant_delays=additional_antenna_delays,
only_complete_pairs=True)
for sname, fpath in raw_fpaths.items() if sname in station_timing_offsets
def get_noise_std(timeID,
initial_block,
max_num_blocks,
max_double_zeros=100,
stations=None,
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt"):
processed_data_folder = processed_data_dir(timeID)
if isinstance(polarization_flips, str):
polarization_flips = read_antenna_pol_flips(processed_data_folder +
'/' + polarization_flips)
if isinstance(processed_data_folder, str):
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
half_window_percent = 0.1
half_window_percent *= 1.1 ## just to be sure we are away from edge
raw_fpaths = filePaths_by_stationName(timeID)
if stations is None:
stations = raw_fpaths.keys()
out_dict = {}
for sname in stations:
print(sname)
TBB_data = MultiFile_Dal1(raw_fpaths[sname],
polarization_flips=polarization_flips,
bad_antennas=bad_antennas)
RFI_filter = window_and_filter(timeID=timeID, sname=sname)
block_size = RFI_filter.blocksize
edge_size = int(half_window_percent * block_size)
antenna_names = TBB_data.get_antenna_names()
measured_std = np.zeros(len(antenna_names))
measured_std[:] = -1
for block_i in range(initial_block, initial_block + max_num_blocks):
for ant_i in range(len(antenna_names)):
if measured_std[ant_i] > 0:
continue ## we got a measurment, so skip it
data = np.array(TBB_data.get_data(block_i * block_size,
block_size,
antenna_index=ant_i),
dtype=np.double)
if num_double_zeros(data) > max_double_zeros:
continue ## bad block, skip
filtered_data = np.real(RFI_filter.filter(data))
filtered_data = filtered_data[edge_size:
-edge_size] ##avoid the edge
measured_std[ant_i] = np.std(filtered_data)
if np.all(measured_std > 0): ## we can break early
break
for ant_name, std in zip(antenna_names, measured_std):
out_dict[ant_name] = std
filter = measured_std > 0
print(" ave std:", np.average(measured_std[filter]))
print(" total ant:", len(filter), "good ant:", np.sum(filter))
print()
return out_dict
def plot_blocks(timeID,
block_size,
block_starts,
guess_delays,
guess_location=None,
bad_stations=[],
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt",
do_remove_saturation=True,
do_remove_RFI=True,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=5,
referance_station="CS002"):
"""plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays,
if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero.
A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations"""
if referance_station in guess_delays:
ref_delay = guess_delays[referance_station]
guess_delays = {
sname: delay - ref_delay
for sname, delay in guess_delays.items()
}
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder +
'/' +
additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in raw_fpaths.keys() if sname not in bad_stations}
if guess_location is not None:
guess_location = np.array(guess_location)
ref_stat_file = raw_data_files[referance_station]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(
ant_loc - guess_location[:3]
) / v_air - ref_stat_file.get_nominal_sample_number() * 5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in guess_delays:
guess_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
guess_delays[sname] += (
np.linalg.norm(ant_loc - guess_location[:3]) / v_air -
ref_delay)
guess_delays[sname] -= data_file.get_nominal_sample_number(
) * 5.0E-9
RFI_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in raw_fpaths.keys() if sname not in bad_stations
}
data = np.empty(block_size, dtype=np.double)
height = 0
t0 = np.arange(block_size) * 5.0E-9
transform = blended_transform_factory(plt.gca().transAxes,
plt.gca().transData)
sname_X_loc = 0.0
for sname, data_file in raw_data_files.items():
print(sname)
station_delay = -data_file.get_nominal_sample_number() * 5.0E-9
if sname in guess_delays:
station_delay = guess_delays[sname]
station_delay_points = int(station_delay / 5.0E-9)
RFI_filter = RFI_filters[sname]
ant_names = data_file.get_antenna_names()
num_antenna_pairs = int(len(ant_names) / 2)
peak_height = 0.0
for point in block_starts:
T = t0 + point * 5.0E-9
for pair in range(num_antenna_pairs):
data[:] = data_file.get_data(point + station_delay_points,
block_size,
antenna_index=pair * 2)
if do_remove_saturation:
remove_saturation(data, positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter(data)
else:
filtered_data = hilbert(data)
even_HE = np.abs(filtered_data)
data[:] = data_file.get_data(point + station_delay_points,
block_size,
antenna_index=pair * 2 + 1)
if do_remove_saturation:
remove_saturation(data, positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter(data)
else:
filtered_data = hilbert(data)
odd_HE = np.abs(filtered_data)
plt.plot(T, even_HE + height, 'r')
plt.plot(T, odd_HE + height, 'g')
max_even = np.max(even_HE)
if max_even > peak_height:
peak_height = max_even
max_odd = np.max(odd_HE)
if max_odd > peak_height:
peak_height = max_odd
# plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
plt.annotate(sname, (sname_X_loc, height),
textcoords=transform,
xycoords=transform)
height += 2 * peak_height
plt.show()
def __init__(self,
timeID,
guess_delays,
block_size,
initial_block,
num_blocks,
working_folder,
other_folder=None,
max_num_stations=np.inf,
guess_location=None,
bad_stations=[],
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt",
do_remove_saturation=True,
do_remove_RFI=True,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
referance_station="CS002",
pulse_length=50,
upsample_factor=4,
min_antenna_amplitude=5,
fix_polarization_delay=True):
self.pressed_data = None
#guess_location = np.array(guess_location[:3])
self.block_size = block_size
self.num_blocks = num_blocks
self.positive_saturation = positive_saturation
self.negative_saturation = negative_saturation
self.saturation_post_removal_length = saturation_post_removal_length
self.saturation_half_hann_length = saturation_half_hann_length
self.do_remove_saturation = do_remove_saturation
self.do_remove_RFI = do_remove_RFI
#### open data ####
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder +
'/' + polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(
processed_data_folder + '/' + additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
station_names = [
sname for sname in raw_fpaths.keys() if sname not in bad_stations
]
self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_names}
if fix_polarization_delay:
for sname, file in self.TBB_files.items():
# delays = file.get_timing_callibration_delays()
# print()
# print()
# print(sname)
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
file.find_and_set_polarization_delay()
# delays = file.get_timing_callibration_delays()
# print("after")
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
#self.RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_names}
self.RFI_filters = {
sname: window_and_filter(timeID=timeID,
sname=sname,
blocksize=block_size)
for sname in station_names
}
#### some data things ####
self.antenna_time_corrections = {
sname: TBB_file.get_total_delays()
for sname, TBB_file in self.TBB_files.items()
}
# ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0]
# for station_corrections in self.antenna_time_corrections.values():
# station_corrections -= ref_antenna_delay
max_num_antennas = np.max(
[len(delays) for delays in self.antenna_time_corrections.values()])
self.temp_data_block = np.empty(
(max_num_antennas, num_blocks, block_size), dtype=np.double)
self.time_array = np.arange(block_size) * 5.0E-9
self.figure = plt.gcf()
self.axes = plt.gca()
#### make managers
guess_delays = {
sname: delay
for sname, delay in guess_delays.items()
if sname not in bad_stations
}
self.clock_offset_manager = clock_offsets(referance_station,
guess_delays, guess_location,
self.TBB_files)
self.frame_manager = frames(initial_block, int(num_blocks / 2),
station_names, max_num_stations,
referance_station, block_size, num_blocks)
rem_sat_curry = cur(remove_saturation, 5)(
positive_saturation=positive_saturation,
negative_saturation=negative_saturation,
post_removal_length=saturation_post_removal_length,
half_hann_length=saturation_half_hann_length)
self.pulse_manager = pulses(
pulse_length=pulse_length,
block_size=block_size,
out_folder=working_folder,
other_folder=other_folder,
TBB_data_dict=self.TBB_files,
used_delay_dict=self.antenna_time_corrections,
upsample_factor=upsample_factor,
min_ant_amp=min_antenna_amplitude,
referance_station=referance_station,
RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None),
remove_saturation_curry=(rem_sat_curry
if self.do_remove_saturation else None))
self.plot()
self.mode = 0 ##0 is change delay, 1 is set peak
self.rect_selector = RectangleSelector(plt.gca(),
self.rectangle_callback,
useblit=True,
rectprops=dict(alpha=0.5,
facecolor='red'),
button=1,
state_modifier_keys={
'move': '',
'clear': '',
'square': '',
'center': ''
})
self.mouse_press = self.figure.canvas.mpl_connect(
'button_press_event', self.mouse_press)
self.mouse_release = plt.gcf().canvas.mpl_connect(
'button_release_event', self.mouse_release)
self.key_press = plt.gcf().canvas.mpl_connect('key_press_event',
self.key_press_event)
self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()]
plt.show()
def plot_blocks(timeID, block_size, block_starts, guess_delays, guess_location = None, bad_stations=[], polarization_flips="polarization_flips.txt",
bad_antennas = "bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", do_remove_saturation = True, do_remove_RFI = True,
positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 5,
referance_station = "CS002", amplify = {}, omitPOL=None):
"""plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays,
if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero.
A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations"""
'''Modified version of plot_multiple_data_all_stations to support usage of clickable GUI for aligning station timings'''
##2
global first_few_names
global defualt_color
global ax
text = []
##~2
if referance_station in guess_delays:
ref_delay = guess_delays[referance_station]
guess_delays = {sname:delay-ref_delay for sname,delay in guess_delays.items()}
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in raw_fpaths.keys() if sname not in bad_stations}
if guess_location is not None:
guess_location = np.array(guess_location)
ref_stat_file = raw_data_files[ referance_station ]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in guess_delays:
guess_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
guess_delays[sname] += (np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_delay)
guess_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9
RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in raw_fpaths.keys() if sname not in bad_stations}
#RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname, blocksize=block_size) for sname in raw_fpaths.keys() if sname not in bad_stations}
data = np.empty(block_size, dtype=np.double)
##3
#fig = plt.figure(figsize=(10, 12))
fig, ax = plt.subplots()
for sname, data_file in raw_data_files.items():
#print("Loading: "+ sname)
if referance_station in sname:
ref_plot = (sname, data_file)
#ref_plot_offset = guess_delays[sname] #currently unsed
else:
plots.append((sname, data_file))
first_few = [ref_plot]
for p in range(0,Nplots-1):
first_few.append(plots.pop(0))
for sname, data_file in first_few:
if sname != referance_station:
first_few_names.append(sname)
##~3
##4
def more_plots(some_data,First,ax,text):
global l,m
ax.clear()
height = 0
t0 = np.arange(block_size)*5.0E-9
transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData)
sname_X_loc = 0.0
n = 0
for sname, data_file in some_data:
print("Plotting: "+sname)
station_delay = -data_file.get_nominal_sample_number()*5.0E-9
if sname in guess_delays:
station_delay = guess_delays[sname]
station_delay_points = int(station_delay/5.0E-9)
RFI_filter = RFI_filters[sname]
ant_names = data_file.get_antenna_names()
num_antenna_pairs = int( len( ant_names )/2 )
peak_height = 0.0
print("Block starts at: "+str(block_starts[0]))
for point in block_starts:
T = t0 + point*5.0E-9
for pair in range(num_antenna_pairs):
data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2)
if do_remove_saturation:
remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter( data )
else:
filtered_data = hilbert(data)
even_HE = np.abs(filtered_data)
data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2+1)
if do_remove_saturation:
remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter( data )
else:
filtered_data = hilbert(data)
odd_HE = np.abs(filtered_data)
#ax.plot(T, even_HE + height, 'r')
#ax.plot(T, odd_HE + height, 'g' )
#for cases where data are hard to align due to signal being tiny:
found = False
if len(amplify) > 0:
for key in amplify:
if sname == key:
if omitPOL != 0:
ax.plot(T, amplify[key]*even_HE + height, 'r')
if omitPOL != 1:
ax.plot(T, amplify[key]*odd_HE + height, 'g' )
#ax.plot(T, amplify[key]*even_HE + height, 'r') twinx with block_starts && point as x rather than t! (can I update to latests plot_multiple?)
#ax.plot(T, amplify[key]*odd_HE + height, 'g' )
found = True
if not found:
if omitPOL != 0:
ax.plot(T, even_HE + height, 'r')
if omitPOL != 1:
ax.plot(T, odd_HE + height, 'g' )
#ax2 = ax.twiny()
#xmin = min(block_starts)
#xmax = max(block_starts)
#ax2.set_xlim(xmin, xmax) #can also use ax2.set_xticks(x)
#ax2.xaxis.set_ticks_position('both')
max_even = np.max(even_HE)
if max_even > peak_height:
peak_height = max_even
max_odd = np.max(odd_HE)
if max_odd > peak_height:
peak_height = max_odd
# plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
plt.sca(ax) #somehow this makes it so that the annotations work on all plots except for the first time "Next stations" is clicked
plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform)
ax.ticklabel_format(useOffset=False)
plt.setp(ax.get_xticklabels(), rotation=60, horizontalalignment='right')
plt.subplots_adjust(bottom=0.1,top=0.98,left=0.21)
if ref_plot_peak_location != 0.0:
print("Setting ref peak to " +str(ref_plot_peak_location))
Cpeak = ref_plot_peak_location #peak center, for plotting
Dpeak = 0.0001 # +/- added to peak center for plotting
plt.xlim(Cpeak-Dpeak,Cpeak+Dpeak)
plt.grid(b=True)
height += 2*peak_height
n+=1
##~4
##5
#######################CallBacks############################
#######################CallBacks############################
#######################CallBacks############################
def onclick(event):
global stations_active
global station_buttons
global ref_plot_peak_location
global set_ref
global set_stations
global default_color
ix = event.xdata
iy = event.ydata
#print("click active stations: ", stations_active)
falsify = []
if set_ref:
ref_plot_peak_location = ix
print("Set ref peak to: "+str(ref_plot_peak_location))
set_ref = False
return #return, don't set anything else!
n=0
for s in stations_active.keys():
if stations_active[s] and s !="":
falsify.append(s)
if s not in guess_delays.keys():
guess_delays[s] = 0.0
print ("Set " + s + " station delay to: ", guess_delays[s] + (ix-ref_plot_peak_location))
tmp_guess_delays[s] = guess_delays[s] + (ix - ref_plot_peak_location)
station_buttons[n].color = default_color
station_buttons[n].hovercolor = default_color
n+=1
if len(falsify)>0:
for f in falsify:
stations_active[f] = False
falsify = []
plt.draw()
def nextCB(event):
global ax
global plots
global station_buttons
global stations_active
global StationCallBacks
print("next!")
stations_active = {}
next_few = [ref_plot]
if len(plots) >= Nplots:
for i in range(0,Nplots-1):
next_few.append(plots.pop(0))
elif len(plots) > 0:
bk = len(plots)
for i in range(0,len(plots)):
next_few.append(plots.pop(0))
for i in range(bk,4):
StationCallBacks[i].reset(i,"")
station_buttons[i].label.set_text("")
else:
next_few = []
for i in range(0,Nplots-1):
StationCallBacks[i].reset(i,"")
station_buttons[i].label.set_text("")
print("All stations timings have been set")
ax.clear()
ax.annotate("All stations timings have been set", (0.5, 0.5))
plt.draw()
return
height = 0.0
n = 0
for sname, data_file in next_few:
if sname != referance_station:
first_few_names.append(sname)
StationCallBacks[n].reset(n,sname)
station_buttons[n].label.set_text("Set "+sname)
stations_active[sname] = False
n+=1
more_plots(next_few,False,ax,text)
plt.draw()
def refpCB(event):
global ref_plot_peak_location
global set_ref
print ("Setting ref peak")# to: ",ix)
set_ref = True
def cCB(text):
Cpeak = float(text) #peak center, for plotting
Dpeak = 0.0001 # +/- added to peak center for plotting
plt.xlim(Cpeak+Dpeak,Cpeak-Dpeak)
plt.draw()
def writeCB(event):
print('Writing guess station delays')
file = open('guess_delays.py','w')
file.write("\n self.guess_timing = " +str(ref_plot_peak_location) + "\n")
#file.write(" self.event_index = ____\n\n")
file.write(' self.guess_station_delays = {\n')
for g in guess_delays:
if g in tmp_guess_delays:
'''If the delay has been updated, write the update:'''
file.write('"'+g+'": '+str(tmp_guess_delays[g])+',\n')
else:
"""Else, just write the old delay:"""
file.write('"'+g+'": '+str(guess_delays[g])+',\n')
file.write('}\n')
print('Done!\n')
def press(event):
val = event.key
print('press', val)
'''if val == ('0' or '1' or '2' or '3' or '4'):
val = int(val)
station_buttons[val].color = 'blue'
station_buttons[val] .hovercolor = 'blue'
'''
class StationCB:
global stations_active
def __init__(self,np,station):
self.N = np
self.station = station
def reset(self,np,station):
self.N = np
self.station = station
def stationCB(self,event):
print("Station active: "+self.station)
#print(self.station + ' You pressed: ',event.name)
#---highlight button with color when pressed state, show buttons in correct order
station_buttons[self.N].color = 'blue'
station_buttons[self.N].hovercolor = 'blue'
stations_active[self.station] = True
######################~CallBacks############################
######################~CallBacks############################
######################~CallBacks############################
more_plots(first_few,True,ax,text)
axnext = plt.axes([ 0.05, 0.80, 0.1, 0.07])
bnext = Button(axnext, 'Next\nStations')
bnext.on_clicked(nextCB)
axfile = plt.axes([0.05,0.7, 0.1, 0.07])
bfile = Button(axfile, 'Write\nGuesses')
bfile.on_clicked(writeCB)
axref = plt.axes([0.05, 0.1, 0.13, 0.07])
bref= Button(axref, 'Set\nRef Peak')
bref.on_clicked(refpCB)
#Cbox = plt.axes([0.05, 0.6, 0.1, 0.07])
#text_box = TextBox(Cbox, 'Peak Center', initial="")
#text_box.on_submit(cCB)
start = 0.22
for f in range(0,Nplots-1):
fax = plt.axes([ 0.05, start+f*0.1, 0.13, 0.05])
fbtn = Button(fax, "Set "+first_few_names[f])
stations_active[first_few_names[f]] = False
s = StationCB(f,first_few_names[f])
StationCallBacks.append(s)
station_axes.append(fax)
station_buttons.append(fbtn)
fbtn.on_clicked(s.stationCB)
fig.canvas.mpl_connect('button_press_event', onclick)
fig.canvas.mpl_connect('key_press_event', press)
plt.show()
def __init__(self,
timeID,
guess_delays,
block_size,
initial_block,
num_blocks,
working_folder,
other_folder=None,
max_num_stations=np.inf,
guess_location=None,
bad_stations=[],
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays=None,
do_remove_saturation=True,
do_remove_RFI=True,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
referance_station="CS002",
pulse_length=50,
upsample_factor=4,
min_antenna_amplitude=5,
fix_polarization_delay=True):
#### disable matplotlib shortcuts
plt.rcParams['keymap.save'] = ''
plt.rcParams['keymap.fullscreen'] = ''
plt.rcParams['keymap.home'] = ''
plt.rcParams['keymap.back'] = ''
plt.rcParams['keymap.forward'] = ''
plt.rcParams['keymap.pan'] = ''
plt.rcParams['keymap.zoom'] = ''
plt.rcParams['keymap.save'] = ''
plt.rcParams['keymap.quit_all'] = ''
plt.rcParams['keymap.grid'] = ''
plt.rcParams['keymap.grid_minor'] = ''
plt.rcParams['keymap.yscale'] = ''
plt.rcParams['keymap.xscale'] = ''
plt.rcParams['keymap.all_axes'] = ''
plt.rcParams['keymap.copy'] = ''
#keymap.help : f1 ## display help about active tools
#keymap.quit : ctrl+w, cmd+w, q ## close the current figure
self.pressed_data = None
#guess_location = np.array(guess_location[:3])
self.block_size = block_size
self.num_blocks = num_blocks
self.positive_saturation = positive_saturation
self.negative_saturation = negative_saturation
self.saturation_post_removal_length = saturation_post_removal_length
self.saturation_half_hann_length = saturation_half_hann_length
self.do_remove_saturation = do_remove_saturation
self.do_remove_RFI = do_remove_RFI
self.referance_station = referance_station
#### open data ####
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder +
'/' + polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
if additional_antenna_delays is not None:
additional_antenna_delays = read_antenna_delays(
processed_data_folder + '/' + additional_antenna_delays)
print(
"WARNING: Additional antenna delays should probably be None!")
raw_fpaths = filePaths_by_stationName(timeID)
station_names = [
sname for sname in raw_fpaths.keys() if sname not in bad_stations
]
self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_names}
if fix_polarization_delay:
for sname, file in self.TBB_files.items():
# delays = file.get_timing_callibration_delays()
# print()
# print()
# print(sname)
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
file.find_and_set_polarization_delay()
# delays = file.get_timing_callibration_delays()
# print("after")
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
self.RFI_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in station_names
}
self.edge_width = int(self.block_size * 0.1)
self.display_block_size = self.block_size - 2 * self.edge_width
#### some data things ####
self.antenna_time_corrections = {
sname: TBB_file.get_total_delays()
for sname, TBB_file in self.TBB_files.items()
}
self.ave_ref_ant_delay = np.average(
self.antenna_time_corrections[referance_station])
# ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0]
# for station_corrections in self.antenna_time_corrections.values():
# station_corrections -= ref_antenna_delay
max_num_antennas = np.max(
[len(delays) for delays in self.antenna_time_corrections.values()])
self.temp_data_block = np.empty(
(max_num_antennas, num_blocks, block_size), dtype=np.double)
self.time_array = np.arange(block_size) * 5.0E-9
self.figure = plt.gcf()
self.axes = plt.gca()
#### make managers
guess_delays = {
sname: delay
for sname, delay in guess_delays.items()
if sname not in bad_stations
}
self.clock_offset_manager = clock_offsets(referance_station,
guess_delays, guess_location,
self.TBB_files)
self.frame_manager = frames(initial_block, int(num_blocks / 2),
station_names, max_num_stations,
referance_station, self.display_block_size,
num_blocks)
rem_sat_curry = cur(remove_saturation, 5)(
positive_saturation=positive_saturation,
negative_saturation=negative_saturation,
post_removal_length=saturation_post_removal_length,
half_hann_length=saturation_half_hann_length)
self.pulse_manager = pulses(
pulse_length=pulse_length,
block_size=block_size,
out_folder=working_folder,
other_folder=other_folder,
TBB_data_dict=self.TBB_files,
used_delay_dict=self.antenna_time_corrections,
upsample_factor=upsample_factor,
min_ant_amp=min_antenna_amplitude,
referance_station=referance_station,
RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None),
remove_saturation_curry=(rem_sat_curry
if self.do_remove_saturation else None))
self.plot()
self.mode = 0 ##0 is change delay, 1 is set peak, etc...
self.input_mode = 0 ## 0 is normal. press 's' to enter 1, where numbers are used to enter event. press 's' again to go back to 0, and search for event
self.search_string = ''
self.rect_selector = RectangleSelector(plt.gca(),
self.rectangle_callback,
useblit=True,
rectprops=dict(alpha=0.5,
facecolor='red'),
button=1,
state_modifier_keys={
'move': '',
'clear': '',
'square': '',
'center': ''
})
self.mouse_press = self.figure.canvas.mpl_connect(
'button_press_event', self.mouse_press)
self.mouse_release = plt.gcf().canvas.mpl_connect(
'button_release_event', self.mouse_release)
self.key_press = plt.gcf().canvas.mpl_connect('key_press_event',
self.key_press_event)
self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()]
plt.show()
def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", max_num_planewaves=np.inf,
positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 50, verbose=True):
left_pulse_length = int(pulse_length/2)
right_pulse_length = pulse_length-left_pulse_length
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = MultiFile_Dal1( raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays )
ant_names = TBB_data.get_antenna_names()
antenna_locations = TBB_data.get_LOFAR_centered_positions()
antenna_delays = TBB_data.get_timing_callibration_delays()
num_antenna_pairs = int( len( ant_names )/2 )
if num_antenna_pairs < min_num_antennas:
return np.array([]), np.array([]), np.array([])
RFI_filter = window_and_filter(timeID=timeID, sname=station)
block_size = RFI_filter.blocksize
out_RMS = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_zenith = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_azimuth = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
N = 0
data = np.empty( (num_antenna_pairs,block_size), dtype=np.double )
for block in range(initial_block, initial_block+number_of_blocks):
if N >= max_num_planewaves:
break
#### open and filter data
for pair_i in range(num_antenna_pairs):
ant_i = pair_i*2 + polarization
data[pair_i,:] = TBB_data.get_data(block*block_size, block_size, antenna_index=ant_i)
remove_saturation(data[pair_i,:], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
np.abs( RFI_filter.filter( data[pair_i,:] ), out=data[pair_i,:])
#### loop over finding planewaves
i = 0
while i < pulses_per_block:
if N >= max_num_planewaves:
break
pulse_location = 0
pulse_amplitude = 0
## find highest peak
for pair_i, HE in enumerate(data):
loc = np.argmax( HE )
amp = HE[ loc ]
if amp > pulse_amplitude:
pulse_amplitude = amp
pulse_location = loc
## check if is strong enough
if pulse_amplitude < min_amplitude:
break
## get
fitter = locatefier()
for pair_i, HE in enumerate(data):
ant_i = pair_i*2 + polarization
signal = np.array( HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] )
if num_double_zeros(signal, threshold=0.1) == 0:
sample_time = 5.0E-9
if upsample_factor > 1:
sample_time /= upsample_factor
signal = resample(signal, len(signal)*upsample_factor )
peak_finder = parabolic_fit( signal )
peak_time = peak_finder.peak_index*sample_time - antenna_delays[ant_i]
fitter.add_antenna(peak_time, antenna_locations[ant_i])
HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] = 0.0
if fitter.num_measurments() < min_num_antennas:
continue
fitter.prep_for_fitting()
X = brute(fitter.RMS, ranges=[[0,np.pi/2],[0,np.pi*2]], Ns=20)
if X[0] >= np.pi/2:
X[0] = (np.pi/2)*0.99
if X[1] >= np.pi*2:
X[1] = (np.pi*2)*0.99
if X[0] < 0:
X[0] = 0.00000001
if X[1] < 0:
X[1] = 0.00000001
ret = least_squares( fitter.time_fits, X, bounds=[[0,0],[np.pi/2,2*np.pi]], ftol=3e-16, xtol=3e-16, gtol=3e-16, x_scale='jac' )
out_RMS[N] = fitter.RMS(ret.x, 3)
out_zenith[N] = ret.x[0]
out_azimuth[N] = ret.x[1]
N += 1
i += 1
return out_RMS[:N], out_zenith[:N], out_azimuth[:N]
def run(self, output_dir, dir_is_organized=True):
processed_data_folder = processed_data_dir( self.timeID )
if dir_is_organized:
output_dir = processed_data_folder +'/' + output_dir
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + self.pol_flips_fname )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + self.bad_antennas_fname )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + self.additional_antenna_delays_fname )
station_timing_offsets = read_station_delays( processed_data_folder+'/'+ self.station_delays_fname )
raw_fpaths = filePaths_by_stationName( self.timeID )
self.station_data_files = []
ref_station_i = None
referance_station_set = None
station_i = 0
self.posix_timestamp = None
for station, fpaths in raw_fpaths.items():
if (station in station_timing_offsets) and (station not in self.stations_to_exclude ):
print('opening', station)
raw_data_file = MultiFile_Dal1(fpaths, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays, pol_flips_are_bad=self.pol_flips_are_bad)
self.station_data_files.append( raw_data_file )
raw_data_file.set_station_delay( station_timing_offsets[station] )
raw_data_file.find_and_set_polarization_delay()
if self.posix_timestamp is None:
self.posix_timestamp = raw_data_file.get_timestamp()
elif self.posix_timestamp != raw_data_file.get_timestamp():
print("ERROR: POSIX timestamps are different")
quit()
if self.referance_station is None:
if (referance_station_set is None) or ( int(station[2:]) < int(referance_station_set[2:]) ): ## if no referance station, use one with smallist number
ref_station_i = station_i
referance_station_set = station
elif self.referance_station == station:
ref_station_i = station_i
referance_station_set = station
station_i += 1
## set reference station first
tmp = self.station_data_files[0]
self.station_data_files[0] = self.station_data_files[ ref_station_i ]
self.station_data_files[ ref_station_i ] = tmp
self.referance_station = referance_station_set
## organize antenana info
self.station_antenna_indeces = [] ## 2D list. First index is station_i, second is a local antenna index, value is station antenna index
self.station_antenna_RMS = [] ## same as above, but value is RMS
self.num_total_antennas = 0
current_ant_i = 0
for station_i,stationFile in enumerate(self.station_data_files):
ant_names = stationFile.get_antenna_names()
num_evenAntennas = int( len(ant_names)/2 )
## get RMS planewave fits
if self.antenna_RMS_info is not None:
antenna_RMS_dict = read_planewave_fits(processed_data_folder+'/'+self.antenna_RMS_info, stationFile.get_station_name() )
else:
antenna_RMS_dict = {}
antenna_RMS_array = np.array([ antenna_RMS_dict[ant_name] if ant_name in antenna_RMS_dict else self.default_expected_RMS for ant_name in ant_names if antName_is_even(ant_name) ])
antenna_indeces = np.arange(num_evenAntennas)*2
## remove bad antennas and sort
ant_is_good = np.isfinite( antenna_RMS_array )
antenna_RMS_array = antenna_RMS_array[ ant_is_good ]
antenna_indeces = antenna_indeces[ ant_is_good ]
sorter = np.argsort( antenna_RMS_array )
antenna_RMS_array = antenna_RMS_array[ sorter ]
antenna_indeces = antenna_indeces[ sorter ]
## select only the best!
antenna_indeces = antenna_indeces[:self.max_antennas_per_station]
antenna_RMS_array = antenna_RMS_array[:self.max_antennas_per_station]
## fill info
self.station_antenna_indeces.append( antenna_indeces )
self.station_antenna_RMS.append( antenna_RMS_array )
self.num_total_antennas += len(antenna_indeces)
current_ant_i += len(antenna_indeces)
### output to header!
if not isdir(output_dir):
mkdir(output_dir)
logging_folder = output_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
header_outfile = h5py.File(output_dir + "/header.h5", "w")
header_outfile.attrs["timeID"] = self.timeID
header_outfile.attrs["initial_datapoint"] = self.initial_datapoint
header_outfile.attrs["max_antennas_per_station"] = self.max_antennas_per_station
header_outfile.attrs["referance_station"] = self.station_data_files[0].get_station_name()
header_outfile.attrs["station_delays_fname"] = self.station_delays_fname
header_outfile.attrs["additional_antenna_delays_fname"] = self.additional_antenna_delays_fname
header_outfile.attrs["bad_antennas_fname"] = self.bad_antennas_fname
header_outfile.attrs["pol_flips_fname"] = self.pol_flips_fname
header_outfile.attrs["blocksize"] = self.blocksize
header_outfile.attrs["remove_saturation"] = self.remove_saturation
header_outfile.attrs["remove_RFI"] = self.remove_RFI
header_outfile.attrs["positive_saturation"] = self.positive_saturation
header_outfile.attrs["negative_saturation"] = self.negative_saturation
header_outfile.attrs["saturation_removal_length"] = self.saturation_removal_length
header_outfile.attrs["saturation_half_hann_length"] = self.saturation_half_hann_length
header_outfile.attrs["hann_window_fraction"] = self.hann_window_fraction
header_outfile.attrs["num_zeros_dataLoss_Threshold"] = self.num_zeros_dataLoss_Threshold
header_outfile.attrs["min_amplitude"] = self.min_amplitude
header_outfile.attrs["upsample_factor"] = self.upsample_factor
header_outfile.attrs["max_events_perBlock"] = self.max_events_perBlock
header_outfile.attrs["min_pulse_length_samples"] = self.min_pulse_length_samples
header_outfile.attrs["erasure_length"] = self.erasure_length
header_outfile.attrs["guess_distance"] = self.guess_distance
header_outfile.attrs["kalman_devations_toSearch"] = self.kalman_devations_toSearch
header_outfile.attrs["pol_flips_are_bad"] = self.pol_flips_are_bad
header_outfile.attrs["antenna_RMS_info"] = self.antenna_RMS_info
header_outfile.attrs["default_expected_RMS"] = self.default_expected_RMS
header_outfile.attrs["max_planewave_RMS"] = self.max_planewave_RMS
header_outfile.attrs["stop_chi_squared"] = self.stop_chi_squared
header_outfile.attrs["max_minimize_itters"] = self.max_minimize_itters
header_outfile.attrs["minimize_ftol"] = self.minimize_ftol
header_outfile.attrs["minimize_xtol"] = self.minimize_xtol
header_outfile.attrs["minimize_gtol"] = self.minimize_gtol
header_outfile.attrs["refStat_delay"] = station_timing_offsets[ self.station_data_files[0].get_station_name() ]
header_outfile.attrs["refStat_timestamp"] = self.posix_timestamp#self.station_data_files[0].get_timestamp()
header_outfile.attrs["refStat_sampleNumber"] = self.station_data_files[0].get_nominal_sample_number()
header_outfile.attrs["stations_to_exclude"] = np.array(self.stations_to_exclude, dtype='S')
header_outfile.attrs["polarization_flips"] = np.array(polarization_flips, dtype='S')
header_outfile.attrs["num_stations"] = len(self.station_data_files)
header_outfile.attrs["num_antennas"] = self.num_total_antennas
for stat_i, (stat_file, antenna_indeces, antenna_RMS) in enumerate(zip(self.station_data_files, self.station_antenna_indeces, self.station_antenna_RMS)):
station_group = header_outfile.create_group( str(stat_i) )
station_group.attrs["sname"] = stat_file.get_station_name()
station_group.attrs["num_antennas"] = len( antenna_indeces )
locations = stat_file.get_LOFAR_centered_positions()
total_delays = stat_file.get_total_delays()
ant_names = stat_file.get_antenna_names()
for ant_i, (stat_index, RMS) in enumerate(zip(antenna_indeces, antenna_RMS)):
antenna_group = station_group.create_group( str(ant_i) )
antenna_group.attrs['antenna_name'] = ant_names[stat_index]
antenna_group.attrs['location'] = locations[stat_index]
antenna_group.attrs['delay'] = total_delays[stat_index]
antenna_group.attrs['planewave_RMS'] = RMS
def open_files(self, log_func=do_nothing, force_metadata_ant_pos=True):
processed_data_loc = processed_data_dir(self.timeID)
#### open callibration data files ####
### TODO: fix these so they are accounted for in a more standard maner
self.station_timing_offsets = read_station_delays(
processed_data_loc + '/' + self.station_delays_fname)
self.additional_ant_delays = read_antenna_delays(
processed_data_loc + '/' + self.additional_antenna_delays_fname)
self.bad_antennas = read_bad_antennas(processed_data_loc + '/' +
self.bad_antennas_fname)
self.pol_flips = read_antenna_pol_flips(processed_data_loc + '/' +
self.pol_flips_fname)
#### open data files, and find RFI ####
raw_fpaths = filePaths_by_stationName(self.timeID)
self.station_names = []
self.input_files = []
self.RFI_filters = []
CS002_index = None
self.station_to_antenna_indeces_dict = {}
for station, fpaths in raw_fpaths.items():
if (station in self.station_timing_offsets) and (
station not in self.stations_to_exclude) and (
self.use_core_stations_S1 or self.use_core_stations_S2
or (not station[:2] == 'CS') or station == "CS002"):
# if (station not in self.stations_to_exclude) and ( self.use_core_stations_S1 or self.use_core_stations_S2 or (not station[:2]=='CS') or station=="CS002" ):
log_func("opening", station)
self.station_names.append(station)
self.station_to_antenna_indeces_dict[station] = []
if station == 'CS002':
CS002_index = len(self.station_names) - 1
new_file = MultiFile_Dal1(
fpaths, force_metadata_ant_pos=force_metadata_ant_pos)
new_file.set_polarization_flips(self.pol_flips)
self.input_files.append(new_file)
RFI_result = None
if self.do_RFI_filtering and self.use_saved_RFI_info:
self.RFI_filters.append(
window_and_filter(timeID=self.timeID, sname=station))
elif self.do_RFI_filtering:
RFI_result = FindRFI(new_file,
self.block_size,
self.initial_RFI_block,
self.RFI_num_blocks,
self.RFI_max_blocks,
verbose=False,
figure_location=None)
self.RFI_filters.append(
window_and_filter(find_RFI=RFI_result))
else: ## only basic filtering
self.RFI_filters.append(
window_and_filter(blocksize=self.block_size))
#### find antenna pairs ####
boundingBox_center = np.average(self.bounding_box, axis=-1)
self.antennas_to_use = pairData_NumAntPerStat(
self.input_files, self.num_antennas_per_station, self.bad_antennas)
# self.antennas_to_use = pairData_NumAntPerStat_PolO(self.input_files, self.num_antennas_per_station, self.bad_antennas )
# self.antennas_to_use = pairData_NumAntPerStat_DualPol(self.input_files, self.num_antennas_per_station, self.bad_antennas )
self.num_antennas = len(self.antennas_to_use)
#### get antenna locations and delays ####
self.antenna_locations = np.zeros((self.num_antennas, 3),
dtype=np.double)
self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double)
self.is_not_core = np.zeros(self.num_antennas, dtype=np.bool)
#### TODO: Do we need this "CSOO2 correction??? will removing it fix our time base?? ####
# clock_corrections = getClockCorrections()
# CS002_correction = -clock_corrections["CS002"] - self.input_files[CS002_index].get_nominal_sample_number()*5.0E-9 ## wierd sign. But is just to correct for previous definiitions
CS002_correction = self.station_timing_offsets[
'CS002'] - self.input_files[CS002_index].get_nominal_sample_number(
) * 5.0E-9 ## wierd sign. But is just to correct for previous definiitions
for ant_i, (station_i,
station_ant_i) in enumerate(self.antennas_to_use):
self.station_to_antenna_indeces_dict[
self.station_names[station_i]].append(
ant_i
) ### TODO: this should probably be a result of pairData_NumAntPerStat
data_file = self.input_files[station_i]
station = self.station_names[station_i]
self.is_not_core[ant_i] = (not station[:2]
== 'CS') or station == "CS002"
ant_name = data_file.get_antenna_names()[station_ant_i]
self.antenna_locations[
ant_i] = data_file.get_LOFAR_centered_positions(
)[station_ant_i]
self.antenna_delays[
ant_i] = data_file.get_timing_callibration_delays(
)[station_ant_i]
## account for station timing offsets
# self.antenna_delays[ant_i] += self.station_timing_offsets[station] + (-clock_corrections[station] - data_file.get_nominal_sample_number()*5.0E-9) - CS002_correction
self.antenna_delays[ant_i] += self.station_timing_offsets[
station] + (-data_file.get_nominal_sample_number() *
5.0E-9) - CS002_correction
## add additional timing delays
##note this only works for even antennas!
if ant_name in self.additional_ant_delays:
self.antenna_delays[ant_i] += self.additional_ant_delays[
ant_name][0]
# #### find prefered station ####
if self.prefered_station is None:
prefered_stat_shortestWindowTime = np.inf
prefered_station_input_file = None
for station, antenna_indeces in self.station_to_antenna_indeces_dict.items(
):
ant_i = antenna_indeces[0] ## just use first antenna for test
if not (self.use_core_stations_S1 or self.is_not_core[ant_i]):
continue
longest_window_time = 0
for ant_j in range(self.num_antennas):
if not (self.use_core_stations_S1
or self.is_not_core[ant_j]):
continue
window_time, throw, throw = find_max_duration(
self.antenna_locations[ant_i],
self.antenna_locations[ant_j],
self.bounding_box,
center=boundingBox_center)
if window_time > longest_window_time:
longest_window_time = window_time
if longest_window_time < prefered_stat_shortestWindowTime:
prefered_stat_shortestWindowTime = longest_window_time
self.prefered_station = station
prefered_station_input_file = self.input_files[
self.antennas_to_use[ant_i][0]]
else: ### TODO: throw error is prefered station is in core, but core not included in stage 1
ant_i = self.station_to_antenna_indeces_dict[
self.prefered_station][0]
prefered_stat_shortestWindowTime = 0
for ant_j in range(self.num_antennas):
if not (self.use_core_stations_S1 or self.is_not_core[ant_j]):
continue
window_time, throw, throw = find_max_duration(
self.antenna_locations[ant_i],
self.antenna_locations[ant_j],
self.bounding_box,
center=boundingBox_center)
if window_time > prefered_stat_shortestWindowTime:
prefered_stat_shortestWindowTime = window_time
prefered_station_input_file = self.input_files[
self.antennas_to_use[ant_i][0]]
log_func("prefered station:", self.prefered_station)
log_func(" max. half window time:",
prefered_stat_shortestWindowTime)
log_func()
log_func()
self.prefered_station_timing_offset = self.station_timing_offsets[
self.
prefered_station] - prefered_station_input_file.get_nominal_sample_number(
) * 5.0E-9
self.prefered_station_antenna_timing_offsets = prefered_station_input_file.get_timing_callibration_delays(
)
self.startBlock_exclusion = int(
0.1 * self.block_size) ### TODO: save these to header.
self.endBlock_exclusion = int(
prefered_stat_shortestWindowTime / 5.0E-9) + 1 + int(
0.1 * self.block_size) ## last bit accounts for hann-window
self.active_block_length = self.block_size - self.startBlock_exclusion - self.endBlock_exclusion ##the length of block used for looking at data
##### find window offsets and lengths ####
self.antenna_data_offsets = np.zeros(self.num_antennas, dtype=np.long)
self.half_antenna_data_length = np.zeros(self.num_antennas,
dtype=np.long)
for ant_i, (station_i,
station_ant_i) in enumerate(self.antennas_to_use):
## now we account for distance to the source
travel_time = np.linalg.norm(self.antenna_locations[ant_i] -
boundingBox_center) / v_air
self.antenna_data_offsets[ant_i] = int(travel_time / 5.0E-9)
### find max duration to any of the prefered antennas
max_duration = 0.0
for prefered_ant_i in self.station_to_antenna_indeces_dict[
self.prefered_station]:
if prefered_ant_i == ant_i:
continue
duration, throw, throw = find_max_duration(
self.antenna_locations[prefered_ant_i],
self.antenna_locations[ant_i], self.bounding_box,
boundingBox_center)
if duration > max_duration:
max_duration = duration
self.half_antenna_data_length[ant_i] = int(
self.pulse_length / 2) + int(duration / 5.0E-9)
self.antenna_data_offsets[ant_i] -= self.half_antenna_data_length[
ant_i]
#### now adjust the data offsets and antenna delays so they are consistent
## first we amount to adjust the offset by time delays mod the sampling time
offset_adjust = int(
self.antenna_delays[ant_i] / 5.0E-9
) ##this needs to be added to offsets and subtracted from delays
## then we can adjust the delays accounting for the data offset
self.antenna_delays[
ant_i] -= self.antenna_data_offsets[ant_i] * 5.0E-9
##now we finally account for large time delays
self.antenna_data_offsets[ant_i] += offset_adjust
self.antenna_delays[ant_i] -= offset_adjust * 5.0E-9
if (not self.use_core_stations_S1) or (not self.use_core_stations_S2):
core_filtered_ant_locs = np.array(
self.antenna_locations[self.is_not_core])
core_filtered_ant_delays = np.array(
self.antenna_delays[self.is_not_core])
#### allocate some memory ####
self.data_block = np.empty((self.num_antennas, self.block_size),
dtype=np.complex)
self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double)
#### initialize stage 1 ####
if self.use_core_stations_S1:
self.trace_length_stage1 = 2 * np.max(
self.half_antenna_data_length)
S1_ant_locs = self.antenna_locations
S1_ant_delays = self.antenna_delays
else:
self.trace_length_stage1 = 2 * np.max(
self.half_antenna_data_length[self.is_not_core])
S1_ant_locs = core_filtered_ant_locs
S1_ant_delays = core_filtered_ant_delays
self.trace_length_stage1 = 2**(int(np.log2(self.trace_length_stage1)) +
1)
self.stage_1_imager = II_tools.image_data_stage1(
S1_ant_locs, S1_ant_delays, self.trace_length_stage1,
self.upsample_factor)
#### initialize stage 2 ####
if self.use_core_stations_S2:
S2_ant_locs = self.antenna_locations
S2_ant_delays = self.antenna_delays
else:
S2_ant_locs = core_filtered_ant_locs
S2_ant_delays = core_filtered_ant_delays
self.trace_length_stage2 = 2**(int(np.log2(self.pulse_length)) + 1)
self.stage_2_window = half_hann_window(self.pulse_length,
self.hann_window_fraction)
self.stage_2_imager = II_tools.image_data_stage2_absBefore(
S2_ant_locs, S2_ant_delays, self.trace_length_stage2,
self.upsample_factor)
self.erasure_window = 1.0 - self.stage_2_window
