def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4, polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays="ant_delays.txt", max_num_planewaves=np.inf, positive_saturation=2046, negative_saturation=-2047, saturation_post_removal_length=50, saturation_half_hann_length=50, verbose=True): """ wrapper around 'planewave_fitter' only present for backwards compatiblility. Returns RMS, zenithal and azimuthal fit values, as three arrays in that order""" processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips) bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas) additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' + additional_antenna_delays) raw_fpaths = filePaths_by_stationName(timeID) TBB_data = MultiFile_Dal1(raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) fitter = planewave_fitter( TBB_data=TBB_data, polarization=polarization, initial_block=initial_block, number_of_blocks=number_of_blocks, pulses_per_block=pulses_per_block, pulse_length=pulse_length, min_amplitude=min_amplitude, upsample_factor=upsample_factor, min_num_antennas=min_num_antennas, max_num_planewaves=max_num_planewaves, verbose=verbose, ## doesn't do anything anyway positive_saturation=positive_saturation, negative_saturation=negative_saturation, saturation_post_removal_length=saturation_post_removal_length, saturation_half_hann_length=saturation_half_hann_length) RMSs, Zeniths, Azimuths, throw = fitter.go_fit() return RMSs, Zeniths, Azimuths
def recalibrate_pulse(timeID, input_fname, output_fname, set_polarization_delay=True, upsample_factor=4, polarization_flips="polarization_flips.txt"): processed_data_folder = processed_data_dir(timeID) raw_fpaths = filePaths_by_stationName(timeID) polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips ) input_file = h5py.File(processed_data_folder+'/'+input_fname, "r") try: output_file = h5py.File(processed_data_folder+'/'+output_fname, "r+") except: output_file = h5py.File(processed_data_folder+'/'+output_fname, "w") sample_time = 5.0e-9 if upsample_factor>1: sample_time /= upsample_factor for sname, h5_statGroup in input_file.items(): out_statGroup = output_file.require_group(sname) print() print() print(sname) datafile = MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips) if set_polarization_delay: datafile.find_and_set_polarization_delay() antenna_calibrations = { antname:calibration for antname,calibration in zip(datafile.get_antenna_names(), datafile.get_total_delays()) } for antname, in_antData in h5_statGroup.items(): out_statGroup.copy(in_antData, out_statGroup, name= antname) out_antData = out_statGroup[antname] old_even = out_antData.attrs['PolE_timeOffset_CS'] old_odd = out_antData.attrs['PolO_timeOffset_CS'] new_even_delay = antenna_calibrations[antname] new_odd_delay = antenna_calibrations[ even_antName_to_odd(antname) ] out_antData.attrs['PolE_timeOffset'] = -new_even_delay ## NOTE: due to historical reasons, there is a sign flip here out_antData.attrs['PolO_timeOffset'] = -new_odd_delay ## NOTE: due to historical reasons, there is a sign flip here out_antData.attrs['PolE_timeOffset_CS'] = new_even_delay out_antData.attrs['PolO_timeOffset_CS'] = new_odd_delay print(antname, old_even-new_even_delay, old_odd-new_odd_delay) starting_index = out_antData.attrs['starting_index'] if np.isfinite( out_antData.attrs['PolE_peakTime'] ): polE_HE = out_antData[1] if upsample_factor>1: polE_HE = resample(polE_HE, len(polE_HE)*upsample_factor ) PolE_peak_finder = parabolic_fit( polE_HE ) out_antData.attrs['PolE_peakTime'] = starting_index*5.0E-9 - new_even_delay + PolE_peak_finder.peak_index*sample_time if np.isfinite( out_antData.attrs['PolO_peakTime'] ): polO_HE = out_antData[3] if upsample_factor>1: polO_HE = resample(polO_HE, len(polO_HE)*upsample_factor ) PolO_peak_finder = parabolic_fit( polO_HE ) out_antData.attrs['PolO_peakTime'] = starting_index*5.0E-9 - new_odd_delay + PolO_peak_finder.peak_index*sample_time
from stokes_utils import zenith_to_src from stokesIO import read_polarization_data, save_acceleration_vector timeID = "D20190424T210306.154Z" utilities.default_raw_data_loc = "/home/student4/Marten/KAP_data_link/lightning_data" utilities.default_processed_data_loc = "/home/student4/Marten/processed_files" processed_data_folder = processed_data_dir(timeID) station_delay_file = "station_delays.txt" polarization_flips = "polarization_flips.txt" bad_antennas = "bad_antennas.txt" additional_antenna_delays = "ant_delays.txt" polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips) bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas) additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' + additional_antenna_delays) station_timing_offsets = read_station_delays(processed_data_folder + '/' + station_delay_file) raw_fpaths = filePaths_by_stationName(timeID) TBB_data = { sname: MultiFile_Dal1(fpath, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays, only_complete_pairs=True)
def get_noise_std(timeID, initial_block, max_num_blocks, max_double_zeros=100, stations=None, polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt"): processed_data_folder = processed_data_dir(timeID) if isinstance(polarization_flips, str): polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips) if isinstance(processed_data_folder, str): bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas) half_window_percent = 0.1 half_window_percent *= 1.1 ## just to be sure we are away from edge raw_fpaths = filePaths_by_stationName(timeID) if stations is None: stations = raw_fpaths.keys() out_dict = {} for sname in stations: print(sname) TBB_data = MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas) RFI_filter = window_and_filter(timeID=timeID, sname=sname) block_size = RFI_filter.blocksize edge_size = int(half_window_percent * block_size) antenna_names = TBB_data.get_antenna_names() measured_std = np.zeros(len(antenna_names)) measured_std[:] = -1 for block_i in range(initial_block, initial_block + max_num_blocks): for ant_i in range(len(antenna_names)): if measured_std[ant_i] > 0: continue ## we got a measurment, so skip it data = np.array(TBB_data.get_data(block_i * block_size, block_size, antenna_index=ant_i), dtype=np.double) if num_double_zeros(data) > max_double_zeros: continue ## bad block, skip filtered_data = np.real(RFI_filter.filter(data)) filtered_data = filtered_data[edge_size: -edge_size] ##avoid the edge measured_std[ant_i] = np.std(filtered_data) if np.all(measured_std > 0): ## we can break early break for ant_name, std in zip(antenna_names, measured_std): out_dict[ant_name] = std filter = measured_std > 0 print(" ave std:", np.average(measured_std[filter])) print(" total ant:", len(filter), "good ant:", np.sum(filter)) print() return out_dict
def plot_blocks(timeID, block_size, block_starts, guess_delays, guess_location=None, bad_stations=[], polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays="ant_delays.txt", do_remove_saturation=True, do_remove_RFI=True, positive_saturation=2046, negative_saturation=-2047, saturation_post_removal_length=50, saturation_half_hann_length=5, referance_station="CS002"): """plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays, if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero. A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations""" if referance_station in guess_delays: ref_delay = guess_delays[referance_station] guess_delays = { sname: delay - ref_delay for sname, delay in guess_delays.items() } processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips) bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas) additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' + additional_antenna_delays) raw_fpaths = filePaths_by_stationName(timeID) raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \ for sname in raw_fpaths.keys() if sname not in bad_stations} if guess_location is not None: guess_location = np.array(guess_location) ref_stat_file = raw_data_files[referance_station] ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0] ref_delay = np.linalg.norm( ant_loc - guess_location[:3] ) / v_air - ref_stat_file.get_nominal_sample_number() * 5.0E-9 for sname, data_file in raw_data_files.items(): if sname not in guess_delays: guess_delays[sname] = 0.0 data_file = raw_data_files[sname] ant_loc = data_file.get_LOFAR_centered_positions()[0] guess_delays[sname] += ( np.linalg.norm(ant_loc - guess_location[:3]) / v_air - ref_delay) guess_delays[sname] -= data_file.get_nominal_sample_number( ) * 5.0E-9 RFI_filters = { sname: window_and_filter(timeID=timeID, sname=sname) for sname in raw_fpaths.keys() if sname not in bad_stations } data = np.empty(block_size, dtype=np.double) height = 0 t0 = np.arange(block_size) * 5.0E-9 transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData) sname_X_loc = 0.0 for sname, data_file in raw_data_files.items(): print(sname) station_delay = -data_file.get_nominal_sample_number() * 5.0E-9 if sname in guess_delays: station_delay = guess_delays[sname] station_delay_points = int(station_delay / 5.0E-9) RFI_filter = RFI_filters[sname] ant_names = data_file.get_antenna_names() num_antenna_pairs = int(len(ant_names) / 2) peak_height = 0.0 for point in block_starts: T = t0 + point * 5.0E-9 for pair in range(num_antenna_pairs): data[:] = data_file.get_data(point + station_delay_points, block_size, antenna_index=pair * 2) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter(data) else: filtered_data = hilbert(data) even_HE = np.abs(filtered_data) data[:] = data_file.get_data(point + station_delay_points, block_size, antenna_index=pair * 2 + 1) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter(data) else: filtered_data = hilbert(data) odd_HE = np.abs(filtered_data) plt.plot(T, even_HE + height, 'r') plt.plot(T, odd_HE + height, 'g') max_even = np.max(even_HE) if max_even > peak_height: peak_height = max_even max_odd = np.max(odd_HE) if max_odd > peak_height: peak_height = max_odd # plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height)) plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform) height += 2 * peak_height plt.show()
def __init__(self, timeID, guess_delays, block_size, initial_block, num_blocks, working_folder, other_folder=None, max_num_stations=np.inf, guess_location=None, bad_stations=[], polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays="ant_delays.txt", do_remove_saturation=True, do_remove_RFI=True, positive_saturation=2046, negative_saturation=-2047, saturation_post_removal_length=50, saturation_half_hann_length=50, referance_station="CS002", pulse_length=50, upsample_factor=4, min_antenna_amplitude=5, fix_polarization_delay=True): self.pressed_data = None #guess_location = np.array(guess_location[:3]) self.block_size = block_size self.num_blocks = num_blocks self.positive_saturation = positive_saturation self.negative_saturation = negative_saturation self.saturation_post_removal_length = saturation_post_removal_length self.saturation_half_hann_length = saturation_half_hann_length self.do_remove_saturation = do_remove_saturation self.do_remove_RFI = do_remove_RFI #### open data #### processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips) bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas) additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays) raw_fpaths = filePaths_by_stationName(timeID) station_names = [ sname for sname in raw_fpaths.keys() if sname not in bad_stations ] self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \ for sname in station_names} if fix_polarization_delay: for sname, file in self.TBB_files.items(): # delays = file.get_timing_callibration_delays() # print() # print() # print(sname) # for even_ant_i in range(0,len(delays),2): # print(" ", delays[even_ant_i+1]-delays[even_ant_i]) file.find_and_set_polarization_delay() # delays = file.get_timing_callibration_delays() # print("after") # for even_ant_i in range(0,len(delays),2): # print(" ", delays[even_ant_i+1]-delays[even_ant_i]) #self.RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_names} self.RFI_filters = { sname: window_and_filter(timeID=timeID, sname=sname, blocksize=block_size) for sname in station_names } #### some data things #### self.antenna_time_corrections = { sname: TBB_file.get_total_delays() for sname, TBB_file in self.TBB_files.items() } # ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0] # for station_corrections in self.antenna_time_corrections.values(): # station_corrections -= ref_antenna_delay max_num_antennas = np.max( [len(delays) for delays in self.antenna_time_corrections.values()]) self.temp_data_block = np.empty( (max_num_antennas, num_blocks, block_size), dtype=np.double) self.time_array = np.arange(block_size) * 5.0E-9 self.figure = plt.gcf() self.axes = plt.gca() #### make managers guess_delays = { sname: delay for sname, delay in guess_delays.items() if sname not in bad_stations } self.clock_offset_manager = clock_offsets(referance_station, guess_delays, guess_location, self.TBB_files) self.frame_manager = frames(initial_block, int(num_blocks / 2), station_names, max_num_stations, referance_station, block_size, num_blocks) rem_sat_curry = cur(remove_saturation, 5)( positive_saturation=positive_saturation, negative_saturation=negative_saturation, post_removal_length=saturation_post_removal_length, half_hann_length=saturation_half_hann_length) self.pulse_manager = pulses( pulse_length=pulse_length, block_size=block_size, out_folder=working_folder, other_folder=other_folder, TBB_data_dict=self.TBB_files, used_delay_dict=self.antenna_time_corrections, upsample_factor=upsample_factor, min_ant_amp=min_antenna_amplitude, referance_station=referance_station, RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None), remove_saturation_curry=(rem_sat_curry if self.do_remove_saturation else None)) self.plot() self.mode = 0 ##0 is change delay, 1 is set peak self.rect_selector = RectangleSelector(plt.gca(), self.rectangle_callback, useblit=True, rectprops=dict(alpha=0.5, facecolor='red'), button=1, state_modifier_keys={ 'move': '', 'clear': '', 'square': '', 'center': '' }) self.mouse_press = self.figure.canvas.mpl_connect( 'button_press_event', self.mouse_press) self.mouse_release = plt.gcf().canvas.mpl_connect( 'button_release_event', self.mouse_release) self.key_press = plt.gcf().canvas.mpl_connect('key_press_event', self.key_press_event) self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()] plt.show()
def plot_blocks(timeID, block_size, block_starts, guess_delays, guess_location = None, bad_stations=[], polarization_flips="polarization_flips.txt", bad_antennas = "bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", do_remove_saturation = True, do_remove_RFI = True, positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 5, referance_station = "CS002", amplify = {}, omitPOL=None): """plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays, if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero. A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations""" '''Modified version of plot_multiple_data_all_stations to support usage of clickable GUI for aligning station timings''' ##2 global first_few_names global defualt_color global ax text = [] ##~2 if referance_station in guess_delays: ref_delay = guess_delays[referance_station] guess_delays = {sname:delay-ref_delay for sname,delay in guess_delays.items()} processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips ) bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas ) additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays ) raw_fpaths = filePaths_by_stationName(timeID) raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \ for sname in raw_fpaths.keys() if sname not in bad_stations} if guess_location is not None: guess_location = np.array(guess_location) ref_stat_file = raw_data_files[ referance_station ] ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0] ref_delay = np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9 for sname, data_file in raw_data_files.items(): if sname not in guess_delays: guess_delays[sname] = 0.0 data_file = raw_data_files[sname] ant_loc = data_file.get_LOFAR_centered_positions()[0] guess_delays[sname] += (np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_delay) guess_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9 RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in raw_fpaths.keys() if sname not in bad_stations} #RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname, blocksize=block_size) for sname in raw_fpaths.keys() if sname not in bad_stations} data = np.empty(block_size, dtype=np.double) ##3 #fig = plt.figure(figsize=(10, 12)) fig, ax = plt.subplots() for sname, data_file in raw_data_files.items(): #print("Loading: "+ sname) if referance_station in sname: ref_plot = (sname, data_file) #ref_plot_offset = guess_delays[sname] #currently unsed else: plots.append((sname, data_file)) first_few = [ref_plot] for p in range(0,Nplots-1): first_few.append(plots.pop(0)) for sname, data_file in first_few: if sname != referance_station: first_few_names.append(sname) ##~3 ##4 def more_plots(some_data,First,ax,text): global l,m ax.clear() height = 0 t0 = np.arange(block_size)*5.0E-9 transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData) sname_X_loc = 0.0 n = 0 for sname, data_file in some_data: print("Plotting: "+sname) station_delay = -data_file.get_nominal_sample_number()*5.0E-9 if sname in guess_delays: station_delay = guess_delays[sname] station_delay_points = int(station_delay/5.0E-9) RFI_filter = RFI_filters[sname] ant_names = data_file.get_antenna_names() num_antenna_pairs = int( len( ant_names )/2 ) peak_height = 0.0 print("Block starts at: "+str(block_starts[0])) for point in block_starts: T = t0 + point*5.0E-9 for pair in range(num_antenna_pairs): data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter( data ) else: filtered_data = hilbert(data) even_HE = np.abs(filtered_data) data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2+1) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter( data ) else: filtered_data = hilbert(data) odd_HE = np.abs(filtered_data) #ax.plot(T, even_HE + height, 'r') #ax.plot(T, odd_HE + height, 'g' ) #for cases where data are hard to align due to signal being tiny: found = False if len(amplify) > 0: for key in amplify: if sname == key: if omitPOL != 0: ax.plot(T, amplify[key]*even_HE + height, 'r') if omitPOL != 1: ax.plot(T, amplify[key]*odd_HE + height, 'g' ) #ax.plot(T, amplify[key]*even_HE + height, 'r') twinx with block_starts && point as x rather than t! (can I update to latests plot_multiple?) #ax.plot(T, amplify[key]*odd_HE + height, 'g' ) found = True if not found: if omitPOL != 0: ax.plot(T, even_HE + height, 'r') if omitPOL != 1: ax.plot(T, odd_HE + height, 'g' ) #ax2 = ax.twiny() #xmin = min(block_starts) #xmax = max(block_starts) #ax2.set_xlim(xmin, xmax) #can also use ax2.set_xticks(x) #ax2.xaxis.set_ticks_position('both') max_even = np.max(even_HE) if max_even > peak_height: peak_height = max_even max_odd = np.max(odd_HE) if max_odd > peak_height: peak_height = max_odd # plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height)) plt.sca(ax) #somehow this makes it so that the annotations work on all plots except for the first time "Next stations" is clicked plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform) ax.ticklabel_format(useOffset=False) plt.setp(ax.get_xticklabels(), rotation=60, horizontalalignment='right') plt.subplots_adjust(bottom=0.1,top=0.98,left=0.21) if ref_plot_peak_location != 0.0: print("Setting ref peak to " +str(ref_plot_peak_location)) Cpeak = ref_plot_peak_location #peak center, for plotting Dpeak = 0.0001 # +/- added to peak center for plotting plt.xlim(Cpeak-Dpeak,Cpeak+Dpeak) plt.grid(b=True) height += 2*peak_height n+=1 ##~4 ##5 #######################CallBacks############################ #######################CallBacks############################ #######################CallBacks############################ def onclick(event): global stations_active global station_buttons global ref_plot_peak_location global set_ref global set_stations global default_color ix = event.xdata iy = event.ydata #print("click active stations: ", stations_active) falsify = [] if set_ref: ref_plot_peak_location = ix print("Set ref peak to: "+str(ref_plot_peak_location)) set_ref = False return #return, don't set anything else! n=0 for s in stations_active.keys(): if stations_active[s] and s !="": falsify.append(s) if s not in guess_delays.keys(): guess_delays[s] = 0.0 print ("Set " + s + " station delay to: ", guess_delays[s] + (ix-ref_plot_peak_location)) tmp_guess_delays[s] = guess_delays[s] + (ix - ref_plot_peak_location) station_buttons[n].color = default_color station_buttons[n].hovercolor = default_color n+=1 if len(falsify)>0: for f in falsify: stations_active[f] = False falsify = [] plt.draw() def nextCB(event): global ax global plots global station_buttons global stations_active global StationCallBacks print("next!") stations_active = {} next_few = [ref_plot] if len(plots) >= Nplots: for i in range(0,Nplots-1): next_few.append(plots.pop(0)) elif len(plots) > 0: bk = len(plots) for i in range(0,len(plots)): next_few.append(plots.pop(0)) for i in range(bk,4): StationCallBacks[i].reset(i,"") station_buttons[i].label.set_text("") else: next_few = [] for i in range(0,Nplots-1): StationCallBacks[i].reset(i,"") station_buttons[i].label.set_text("") print("All stations timings have been set") ax.clear() ax.annotate("All stations timings have been set", (0.5, 0.5)) plt.draw() return height = 0.0 n = 0 for sname, data_file in next_few: if sname != referance_station: first_few_names.append(sname) StationCallBacks[n].reset(n,sname) station_buttons[n].label.set_text("Set "+sname) stations_active[sname] = False n+=1 more_plots(next_few,False,ax,text) plt.draw() def refpCB(event): global ref_plot_peak_location global set_ref print ("Setting ref peak")# to: ",ix) set_ref = True def cCB(text): Cpeak = float(text) #peak center, for plotting Dpeak = 0.0001 # +/- added to peak center for plotting plt.xlim(Cpeak+Dpeak,Cpeak-Dpeak) plt.draw() def writeCB(event): print('Writing guess station delays') file = open('guess_delays.py','w') file.write("\n self.guess_timing = " +str(ref_plot_peak_location) + "\n") #file.write(" self.event_index = ____\n\n") file.write(' self.guess_station_delays = {\n') for g in guess_delays: if g in tmp_guess_delays: '''If the delay has been updated, write the update:''' file.write('"'+g+'": '+str(tmp_guess_delays[g])+',\n') else: """Else, just write the old delay:""" file.write('"'+g+'": '+str(guess_delays[g])+',\n') file.write('}\n') print('Done!\n') def press(event): val = event.key print('press', val) '''if val == ('0' or '1' or '2' or '3' or '4'): val = int(val) station_buttons[val].color = 'blue' station_buttons[val] .hovercolor = 'blue' ''' class StationCB: global stations_active def __init__(self,np,station): self.N = np self.station = station def reset(self,np,station): self.N = np self.station = station def stationCB(self,event): print("Station active: "+self.station) #print(self.station + ' You pressed: ',event.name) #---highlight button with color when pressed state, show buttons in correct order station_buttons[self.N].color = 'blue' station_buttons[self.N].hovercolor = 'blue' stations_active[self.station] = True ######################~CallBacks############################ ######################~CallBacks############################ ######################~CallBacks############################ more_plots(first_few,True,ax,text) axnext = plt.axes([ 0.05, 0.80, 0.1, 0.07]) bnext = Button(axnext, 'Next\nStations') bnext.on_clicked(nextCB) axfile = plt.axes([0.05,0.7, 0.1, 0.07]) bfile = Button(axfile, 'Write\nGuesses') bfile.on_clicked(writeCB) axref = plt.axes([0.05, 0.1, 0.13, 0.07]) bref= Button(axref, 'Set\nRef Peak') bref.on_clicked(refpCB) #Cbox = plt.axes([0.05, 0.6, 0.1, 0.07]) #text_box = TextBox(Cbox, 'Peak Center', initial="") #text_box.on_submit(cCB) start = 0.22 for f in range(0,Nplots-1): fax = plt.axes([ 0.05, start+f*0.1, 0.13, 0.05]) fbtn = Button(fax, "Set "+first_few_names[f]) stations_active[first_few_names[f]] = False s = StationCB(f,first_few_names[f]) StationCallBacks.append(s) station_axes.append(fax) station_buttons.append(fbtn) fbtn.on_clicked(s.stationCB) fig.canvas.mpl_connect('button_press_event', onclick) fig.canvas.mpl_connect('key_press_event', press) plt.show()
def __init__(self, timeID, guess_delays, block_size, initial_block, num_blocks, working_folder, other_folder=None, max_num_stations=np.inf, guess_location=None, bad_stations=[], polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays=None, do_remove_saturation=True, do_remove_RFI=True, positive_saturation=2046, negative_saturation=-2047, saturation_post_removal_length=50, saturation_half_hann_length=50, referance_station="CS002", pulse_length=50, upsample_factor=4, min_antenna_amplitude=5, fix_polarization_delay=True): #### disable matplotlib shortcuts plt.rcParams['keymap.save'] = '' plt.rcParams['keymap.fullscreen'] = '' plt.rcParams['keymap.home'] = '' plt.rcParams['keymap.back'] = '' plt.rcParams['keymap.forward'] = '' plt.rcParams['keymap.pan'] = '' plt.rcParams['keymap.zoom'] = '' plt.rcParams['keymap.save'] = '' plt.rcParams['keymap.quit_all'] = '' plt.rcParams['keymap.grid'] = '' plt.rcParams['keymap.grid_minor'] = '' plt.rcParams['keymap.yscale'] = '' plt.rcParams['keymap.xscale'] = '' plt.rcParams['keymap.all_axes'] = '' plt.rcParams['keymap.copy'] = '' #keymap.help : f1 ## display help about active tools #keymap.quit : ctrl+w, cmd+w, q ## close the current figure self.pressed_data = None #guess_location = np.array(guess_location[:3]) self.block_size = block_size self.num_blocks = num_blocks self.positive_saturation = positive_saturation self.negative_saturation = negative_saturation self.saturation_post_removal_length = saturation_post_removal_length self.saturation_half_hann_length = saturation_half_hann_length self.do_remove_saturation = do_remove_saturation self.do_remove_RFI = do_remove_RFI self.referance_station = referance_station #### open data #### processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips) bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas) if additional_antenna_delays is not None: additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays) print( "WARNING: Additional antenna delays should probably be None!") raw_fpaths = filePaths_by_stationName(timeID) station_names = [ sname for sname in raw_fpaths.keys() if sname not in bad_stations ] self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \ for sname in station_names} if fix_polarization_delay: for sname, file in self.TBB_files.items(): # delays = file.get_timing_callibration_delays() # print() # print() # print(sname) # for even_ant_i in range(0,len(delays),2): # print(" ", delays[even_ant_i+1]-delays[even_ant_i]) file.find_and_set_polarization_delay() # delays = file.get_timing_callibration_delays() # print("after") # for even_ant_i in range(0,len(delays),2): # print(" ", delays[even_ant_i+1]-delays[even_ant_i]) self.RFI_filters = { sname: window_and_filter(timeID=timeID, sname=sname) for sname in station_names } self.edge_width = int(self.block_size * 0.1) self.display_block_size = self.block_size - 2 * self.edge_width #### some data things #### self.antenna_time_corrections = { sname: TBB_file.get_total_delays() for sname, TBB_file in self.TBB_files.items() } self.ave_ref_ant_delay = np.average( self.antenna_time_corrections[referance_station]) # ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0] # for station_corrections in self.antenna_time_corrections.values(): # station_corrections -= ref_antenna_delay max_num_antennas = np.max( [len(delays) for delays in self.antenna_time_corrections.values()]) self.temp_data_block = np.empty( (max_num_antennas, num_blocks, block_size), dtype=np.double) self.time_array = np.arange(block_size) * 5.0E-9 self.figure = plt.gcf() self.axes = plt.gca() #### make managers guess_delays = { sname: delay for sname, delay in guess_delays.items() if sname not in bad_stations } self.clock_offset_manager = clock_offsets(referance_station, guess_delays, guess_location, self.TBB_files) self.frame_manager = frames(initial_block, int(num_blocks / 2), station_names, max_num_stations, referance_station, self.display_block_size, num_blocks) rem_sat_curry = cur(remove_saturation, 5)( positive_saturation=positive_saturation, negative_saturation=negative_saturation, post_removal_length=saturation_post_removal_length, half_hann_length=saturation_half_hann_length) self.pulse_manager = pulses( pulse_length=pulse_length, block_size=block_size, out_folder=working_folder, other_folder=other_folder, TBB_data_dict=self.TBB_files, used_delay_dict=self.antenna_time_corrections, upsample_factor=upsample_factor, min_ant_amp=min_antenna_amplitude, referance_station=referance_station, RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None), remove_saturation_curry=(rem_sat_curry if self.do_remove_saturation else None)) self.plot() self.mode = 0 ##0 is change delay, 1 is set peak, etc... self.input_mode = 0 ## 0 is normal. press 's' to enter 1, where numbers are used to enter event. press 's' again to go back to 0, and search for event self.search_string = '' self.rect_selector = RectangleSelector(plt.gca(), self.rectangle_callback, useblit=True, rectprops=dict(alpha=0.5, facecolor='red'), button=1, state_modifier_keys={ 'move': '', 'clear': '', 'square': '', 'center': '' }) self.mouse_press = self.figure.canvas.mpl_connect( 'button_press_event', self.mouse_press) self.mouse_release = plt.gcf().canvas.mpl_connect( 'button_release_event', self.mouse_release) self.key_press = plt.gcf().canvas.mpl_connect('key_press_event', self.key_press_event) self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()] plt.show()
def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4, polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", max_num_planewaves=np.inf, positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 50, verbose=True): left_pulse_length = int(pulse_length/2) right_pulse_length = pulse_length-left_pulse_length processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips ) bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas ) additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays ) raw_fpaths = filePaths_by_stationName(timeID) TBB_data = MultiFile_Dal1( raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays ) ant_names = TBB_data.get_antenna_names() antenna_locations = TBB_data.get_LOFAR_centered_positions() antenna_delays = TBB_data.get_timing_callibration_delays() num_antenna_pairs = int( len( ant_names )/2 ) if num_antenna_pairs < min_num_antennas: return np.array([]), np.array([]), np.array([]) RFI_filter = window_and_filter(timeID=timeID, sname=station) block_size = RFI_filter.blocksize out_RMS = np.empty( number_of_blocks*pulses_per_block, dtype=np.double ) out_zenith = np.empty( number_of_blocks*pulses_per_block, dtype=np.double ) out_azimuth = np.empty( number_of_blocks*pulses_per_block, dtype=np.double ) N = 0 data = np.empty( (num_antenna_pairs,block_size), dtype=np.double ) for block in range(initial_block, initial_block+number_of_blocks): if N >= max_num_planewaves: break #### open and filter data for pair_i in range(num_antenna_pairs): ant_i = pair_i*2 + polarization data[pair_i,:] = TBB_data.get_data(block*block_size, block_size, antenna_index=ant_i) remove_saturation(data[pair_i,:], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) np.abs( RFI_filter.filter( data[pair_i,:] ), out=data[pair_i,:]) #### loop over finding planewaves i = 0 while i < pulses_per_block: if N >= max_num_planewaves: break pulse_location = 0 pulse_amplitude = 0 ## find highest peak for pair_i, HE in enumerate(data): loc = np.argmax( HE ) amp = HE[ loc ] if amp > pulse_amplitude: pulse_amplitude = amp pulse_location = loc ## check if is strong enough if pulse_amplitude < min_amplitude: break ## get fitter = locatefier() for pair_i, HE in enumerate(data): ant_i = pair_i*2 + polarization signal = np.array( HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] ) if num_double_zeros(signal, threshold=0.1) == 0: sample_time = 5.0E-9 if upsample_factor > 1: sample_time /= upsample_factor signal = resample(signal, len(signal)*upsample_factor ) peak_finder = parabolic_fit( signal ) peak_time = peak_finder.peak_index*sample_time - antenna_delays[ant_i] fitter.add_antenna(peak_time, antenna_locations[ant_i]) HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] = 0.0 if fitter.num_measurments() < min_num_antennas: continue fitter.prep_for_fitting() X = brute(fitter.RMS, ranges=[[0,np.pi/2],[0,np.pi*2]], Ns=20) if X[0] >= np.pi/2: X[0] = (np.pi/2)*0.99 if X[1] >= np.pi*2: X[1] = (np.pi*2)*0.99 if X[0] < 0: X[0] = 0.00000001 if X[1] < 0: X[1] = 0.00000001 ret = least_squares( fitter.time_fits, X, bounds=[[0,0],[np.pi/2,2*np.pi]], ftol=3e-16, xtol=3e-16, gtol=3e-16, x_scale='jac' ) out_RMS[N] = fitter.RMS(ret.x, 3) out_zenith[N] = ret.x[0] out_azimuth[N] = ret.x[1] N += 1 i += 1 return out_RMS[:N], out_zenith[:N], out_azimuth[:N]
def run(self, output_dir, dir_is_organized=True): processed_data_folder = processed_data_dir( self.timeID ) if dir_is_organized: output_dir = processed_data_folder +'/' + output_dir polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + self.pol_flips_fname ) bad_antennas = read_bad_antennas( processed_data_folder + '/' + self.bad_antennas_fname ) additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + self.additional_antenna_delays_fname ) station_timing_offsets = read_station_delays( processed_data_folder+'/'+ self.station_delays_fname ) raw_fpaths = filePaths_by_stationName( self.timeID ) self.station_data_files = [] ref_station_i = None referance_station_set = None station_i = 0 self.posix_timestamp = None for station, fpaths in raw_fpaths.items(): if (station in station_timing_offsets) and (station not in self.stations_to_exclude ): print('opening', station) raw_data_file = MultiFile_Dal1(fpaths, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays, pol_flips_are_bad=self.pol_flips_are_bad) self.station_data_files.append( raw_data_file ) raw_data_file.set_station_delay( station_timing_offsets[station] ) raw_data_file.find_and_set_polarization_delay() if self.posix_timestamp is None: self.posix_timestamp = raw_data_file.get_timestamp() elif self.posix_timestamp != raw_data_file.get_timestamp(): print("ERROR: POSIX timestamps are different") quit() if self.referance_station is None: if (referance_station_set is None) or ( int(station[2:]) < int(referance_station_set[2:]) ): ## if no referance station, use one with smallist number ref_station_i = station_i referance_station_set = station elif self.referance_station == station: ref_station_i = station_i referance_station_set = station station_i += 1 ## set reference station first tmp = self.station_data_files[0] self.station_data_files[0] = self.station_data_files[ ref_station_i ] self.station_data_files[ ref_station_i ] = tmp self.referance_station = referance_station_set ## organize antenana info self.station_antenna_indeces = [] ## 2D list. First index is station_i, second is a local antenna index, value is station antenna index self.station_antenna_RMS = [] ## same as above, but value is RMS self.num_total_antennas = 0 current_ant_i = 0 for station_i,stationFile in enumerate(self.station_data_files): ant_names = stationFile.get_antenna_names() num_evenAntennas = int( len(ant_names)/2 ) ## get RMS planewave fits if self.antenna_RMS_info is not None: antenna_RMS_dict = read_planewave_fits(processed_data_folder+'/'+self.antenna_RMS_info, stationFile.get_station_name() ) else: antenna_RMS_dict = {} antenna_RMS_array = np.array([ antenna_RMS_dict[ant_name] if ant_name in antenna_RMS_dict else self.default_expected_RMS for ant_name in ant_names if antName_is_even(ant_name) ]) antenna_indeces = np.arange(num_evenAntennas)*2 ## remove bad antennas and sort ant_is_good = np.isfinite( antenna_RMS_array ) antenna_RMS_array = antenna_RMS_array[ ant_is_good ] antenna_indeces = antenna_indeces[ ant_is_good ] sorter = np.argsort( antenna_RMS_array ) antenna_RMS_array = antenna_RMS_array[ sorter ] antenna_indeces = antenna_indeces[ sorter ] ## select only the best! antenna_indeces = antenna_indeces[:self.max_antennas_per_station] antenna_RMS_array = antenna_RMS_array[:self.max_antennas_per_station] ## fill info self.station_antenna_indeces.append( antenna_indeces ) self.station_antenna_RMS.append( antenna_RMS_array ) self.num_total_antennas += len(antenna_indeces) current_ant_i += len(antenna_indeces) ### output to header! if not isdir(output_dir): mkdir(output_dir) logging_folder = output_dir + '/logs_and_plots' if not isdir(logging_folder): mkdir(logging_folder) header_outfile = h5py.File(output_dir + "/header.h5", "w") header_outfile.attrs["timeID"] = self.timeID header_outfile.attrs["initial_datapoint"] = self.initial_datapoint header_outfile.attrs["max_antennas_per_station"] = self.max_antennas_per_station header_outfile.attrs["referance_station"] = self.station_data_files[0].get_station_name() header_outfile.attrs["station_delays_fname"] = self.station_delays_fname header_outfile.attrs["additional_antenna_delays_fname"] = self.additional_antenna_delays_fname header_outfile.attrs["bad_antennas_fname"] = self.bad_antennas_fname header_outfile.attrs["pol_flips_fname"] = self.pol_flips_fname header_outfile.attrs["blocksize"] = self.blocksize header_outfile.attrs["remove_saturation"] = self.remove_saturation header_outfile.attrs["remove_RFI"] = self.remove_RFI header_outfile.attrs["positive_saturation"] = self.positive_saturation header_outfile.attrs["negative_saturation"] = self.negative_saturation header_outfile.attrs["saturation_removal_length"] = self.saturation_removal_length header_outfile.attrs["saturation_half_hann_length"] = self.saturation_half_hann_length header_outfile.attrs["hann_window_fraction"] = self.hann_window_fraction header_outfile.attrs["num_zeros_dataLoss_Threshold"] = self.num_zeros_dataLoss_Threshold header_outfile.attrs["min_amplitude"] = self.min_amplitude header_outfile.attrs["upsample_factor"] = self.upsample_factor header_outfile.attrs["max_events_perBlock"] = self.max_events_perBlock header_outfile.attrs["min_pulse_length_samples"] = self.min_pulse_length_samples header_outfile.attrs["erasure_length"] = self.erasure_length header_outfile.attrs["guess_distance"] = self.guess_distance header_outfile.attrs["kalman_devations_toSearch"] = self.kalman_devations_toSearch header_outfile.attrs["pol_flips_are_bad"] = self.pol_flips_are_bad header_outfile.attrs["antenna_RMS_info"] = self.antenna_RMS_info header_outfile.attrs["default_expected_RMS"] = self.default_expected_RMS header_outfile.attrs["max_planewave_RMS"] = self.max_planewave_RMS header_outfile.attrs["stop_chi_squared"] = self.stop_chi_squared header_outfile.attrs["max_minimize_itters"] = self.max_minimize_itters header_outfile.attrs["minimize_ftol"] = self.minimize_ftol header_outfile.attrs["minimize_xtol"] = self.minimize_xtol header_outfile.attrs["minimize_gtol"] = self.minimize_gtol header_outfile.attrs["refStat_delay"] = station_timing_offsets[ self.station_data_files[0].get_station_name() ] header_outfile.attrs["refStat_timestamp"] = self.posix_timestamp#self.station_data_files[0].get_timestamp() header_outfile.attrs["refStat_sampleNumber"] = self.station_data_files[0].get_nominal_sample_number() header_outfile.attrs["stations_to_exclude"] = np.array(self.stations_to_exclude, dtype='S') header_outfile.attrs["polarization_flips"] = np.array(polarization_flips, dtype='S') header_outfile.attrs["num_stations"] = len(self.station_data_files) header_outfile.attrs["num_antennas"] = self.num_total_antennas for stat_i, (stat_file, antenna_indeces, antenna_RMS) in enumerate(zip(self.station_data_files, self.station_antenna_indeces, self.station_antenna_RMS)): station_group = header_outfile.create_group( str(stat_i) ) station_group.attrs["sname"] = stat_file.get_station_name() station_group.attrs["num_antennas"] = len( antenna_indeces ) locations = stat_file.get_LOFAR_centered_positions() total_delays = stat_file.get_total_delays() ant_names = stat_file.get_antenna_names() for ant_i, (stat_index, RMS) in enumerate(zip(antenna_indeces, antenna_RMS)): antenna_group = station_group.create_group( str(ant_i) ) antenna_group.attrs['antenna_name'] = ant_names[stat_index] antenna_group.attrs['location'] = locations[stat_index] antenna_group.attrs['delay'] = total_delays[stat_index] antenna_group.attrs['planewave_RMS'] = RMS
def open_files(self, log_func=do_nothing, force_metadata_ant_pos=True): processed_data_loc = processed_data_dir(self.timeID) #### open callibration data files #### ### TODO: fix these so they are accounted for in a more standard maner self.station_timing_offsets = read_station_delays( processed_data_loc + '/' + self.station_delays_fname) self.additional_ant_delays = read_antenna_delays( processed_data_loc + '/' + self.additional_antenna_delays_fname) self.bad_antennas = read_bad_antennas(processed_data_loc + '/' + self.bad_antennas_fname) self.pol_flips = read_antenna_pol_flips(processed_data_loc + '/' + self.pol_flips_fname) #### open data files, and find RFI #### raw_fpaths = filePaths_by_stationName(self.timeID) self.station_names = [] self.input_files = [] self.RFI_filters = [] CS002_index = None self.station_to_antenna_indeces_dict = {} for station, fpaths in raw_fpaths.items(): if (station in self.station_timing_offsets) and ( station not in self.stations_to_exclude) and ( self.use_core_stations_S1 or self.use_core_stations_S2 or (not station[:2] == 'CS') or station == "CS002"): # if (station not in self.stations_to_exclude) and ( self.use_core_stations_S1 or self.use_core_stations_S2 or (not station[:2]=='CS') or station=="CS002" ): log_func("opening", station) self.station_names.append(station) self.station_to_antenna_indeces_dict[station] = [] if station == 'CS002': CS002_index = len(self.station_names) - 1 new_file = MultiFile_Dal1( fpaths, force_metadata_ant_pos=force_metadata_ant_pos) new_file.set_polarization_flips(self.pol_flips) self.input_files.append(new_file) RFI_result = None if self.do_RFI_filtering and self.use_saved_RFI_info: self.RFI_filters.append( window_and_filter(timeID=self.timeID, sname=station)) elif self.do_RFI_filtering: RFI_result = FindRFI(new_file, self.block_size, self.initial_RFI_block, self.RFI_num_blocks, self.RFI_max_blocks, verbose=False, figure_location=None) self.RFI_filters.append( window_and_filter(find_RFI=RFI_result)) else: ## only basic filtering self.RFI_filters.append( window_and_filter(blocksize=self.block_size)) #### find antenna pairs #### boundingBox_center = np.average(self.bounding_box, axis=-1) self.antennas_to_use = pairData_NumAntPerStat( self.input_files, self.num_antennas_per_station, self.bad_antennas) # self.antennas_to_use = pairData_NumAntPerStat_PolO(self.input_files, self.num_antennas_per_station, self.bad_antennas ) # self.antennas_to_use = pairData_NumAntPerStat_DualPol(self.input_files, self.num_antennas_per_station, self.bad_antennas ) self.num_antennas = len(self.antennas_to_use) #### get antenna locations and delays #### self.antenna_locations = np.zeros((self.num_antennas, 3), dtype=np.double) self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double) self.is_not_core = np.zeros(self.num_antennas, dtype=np.bool) #### TODO: Do we need this "CSOO2 correction??? will removing it fix our time base?? #### # clock_corrections = getClockCorrections() # CS002_correction = -clock_corrections["CS002"] - self.input_files[CS002_index].get_nominal_sample_number()*5.0E-9 ## wierd sign. But is just to correct for previous definiitions CS002_correction = self.station_timing_offsets[ 'CS002'] - self.input_files[CS002_index].get_nominal_sample_number( ) * 5.0E-9 ## wierd sign. But is just to correct for previous definiitions for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use): self.station_to_antenna_indeces_dict[ self.station_names[station_i]].append( ant_i ) ### TODO: this should probably be a result of pairData_NumAntPerStat data_file = self.input_files[station_i] station = self.station_names[station_i] self.is_not_core[ant_i] = (not station[:2] == 'CS') or station == "CS002" ant_name = data_file.get_antenna_names()[station_ant_i] self.antenna_locations[ ant_i] = data_file.get_LOFAR_centered_positions( )[station_ant_i] self.antenna_delays[ ant_i] = data_file.get_timing_callibration_delays( )[station_ant_i] ## account for station timing offsets # self.antenna_delays[ant_i] += self.station_timing_offsets[station] + (-clock_corrections[station] - data_file.get_nominal_sample_number()*5.0E-9) - CS002_correction self.antenna_delays[ant_i] += self.station_timing_offsets[ station] + (-data_file.get_nominal_sample_number() * 5.0E-9) - CS002_correction ## add additional timing delays ##note this only works for even antennas! if ant_name in self.additional_ant_delays: self.antenna_delays[ant_i] += self.additional_ant_delays[ ant_name][0] # #### find prefered station #### if self.prefered_station is None: prefered_stat_shortestWindowTime = np.inf prefered_station_input_file = None for station, antenna_indeces in self.station_to_antenna_indeces_dict.items( ): ant_i = antenna_indeces[0] ## just use first antenna for test if not (self.use_core_stations_S1 or self.is_not_core[ant_i]): continue longest_window_time = 0 for ant_j in range(self.num_antennas): if not (self.use_core_stations_S1 or self.is_not_core[ant_j]): continue window_time, throw, throw = find_max_duration( self.antenna_locations[ant_i], self.antenna_locations[ant_j], self.bounding_box, center=boundingBox_center) if window_time > longest_window_time: longest_window_time = window_time if longest_window_time < prefered_stat_shortestWindowTime: prefered_stat_shortestWindowTime = longest_window_time self.prefered_station = station prefered_station_input_file = self.input_files[ self.antennas_to_use[ant_i][0]] else: ### TODO: throw error is prefered station is in core, but core not included in stage 1 ant_i = self.station_to_antenna_indeces_dict[ self.prefered_station][0] prefered_stat_shortestWindowTime = 0 for ant_j in range(self.num_antennas): if not (self.use_core_stations_S1 or self.is_not_core[ant_j]): continue window_time, throw, throw = find_max_duration( self.antenna_locations[ant_i], self.antenna_locations[ant_j], self.bounding_box, center=boundingBox_center) if window_time > prefered_stat_shortestWindowTime: prefered_stat_shortestWindowTime = window_time prefered_station_input_file = self.input_files[ self.antennas_to_use[ant_i][0]] log_func("prefered station:", self.prefered_station) log_func(" max. half window time:", prefered_stat_shortestWindowTime) log_func() log_func() self.prefered_station_timing_offset = self.station_timing_offsets[ self. prefered_station] - prefered_station_input_file.get_nominal_sample_number( ) * 5.0E-9 self.prefered_station_antenna_timing_offsets = prefered_station_input_file.get_timing_callibration_delays( ) self.startBlock_exclusion = int( 0.1 * self.block_size) ### TODO: save these to header. self.endBlock_exclusion = int( prefered_stat_shortestWindowTime / 5.0E-9) + 1 + int( 0.1 * self.block_size) ## last bit accounts for hann-window self.active_block_length = self.block_size - self.startBlock_exclusion - self.endBlock_exclusion ##the length of block used for looking at data ##### find window offsets and lengths #### self.antenna_data_offsets = np.zeros(self.num_antennas, dtype=np.long) self.half_antenna_data_length = np.zeros(self.num_antennas, dtype=np.long) for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use): ## now we account for distance to the source travel_time = np.linalg.norm(self.antenna_locations[ant_i] - boundingBox_center) / v_air self.antenna_data_offsets[ant_i] = int(travel_time / 5.0E-9) ### find max duration to any of the prefered antennas max_duration = 0.0 for prefered_ant_i in self.station_to_antenna_indeces_dict[ self.prefered_station]: if prefered_ant_i == ant_i: continue duration, throw, throw = find_max_duration( self.antenna_locations[prefered_ant_i], self.antenna_locations[ant_i], self.bounding_box, boundingBox_center) if duration > max_duration: max_duration = duration self.half_antenna_data_length[ant_i] = int( self.pulse_length / 2) + int(duration / 5.0E-9) self.antenna_data_offsets[ant_i] -= self.half_antenna_data_length[ ant_i] #### now adjust the data offsets and antenna delays so they are consistent ## first we amount to adjust the offset by time delays mod the sampling time offset_adjust = int( self.antenna_delays[ant_i] / 5.0E-9 ) ##this needs to be added to offsets and subtracted from delays ## then we can adjust the delays accounting for the data offset self.antenna_delays[ ant_i] -= self.antenna_data_offsets[ant_i] * 5.0E-9 ##now we finally account for large time delays self.antenna_data_offsets[ant_i] += offset_adjust self.antenna_delays[ant_i] -= offset_adjust * 5.0E-9 if (not self.use_core_stations_S1) or (not self.use_core_stations_S2): core_filtered_ant_locs = np.array( self.antenna_locations[self.is_not_core]) core_filtered_ant_delays = np.array( self.antenna_delays[self.is_not_core]) #### allocate some memory #### self.data_block = np.empty((self.num_antennas, self.block_size), dtype=np.complex) self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double) #### initialize stage 1 #### if self.use_core_stations_S1: self.trace_length_stage1 = 2 * np.max( self.half_antenna_data_length) S1_ant_locs = self.antenna_locations S1_ant_delays = self.antenna_delays else: self.trace_length_stage1 = 2 * np.max( self.half_antenna_data_length[self.is_not_core]) S1_ant_locs = core_filtered_ant_locs S1_ant_delays = core_filtered_ant_delays self.trace_length_stage1 = 2**(int(np.log2(self.trace_length_stage1)) + 1) self.stage_1_imager = II_tools.image_data_stage1( S1_ant_locs, S1_ant_delays, self.trace_length_stage1, self.upsample_factor) #### initialize stage 2 #### if self.use_core_stations_S2: S2_ant_locs = self.antenna_locations S2_ant_delays = self.antenna_delays else: S2_ant_locs = core_filtered_ant_locs S2_ant_delays = core_filtered_ant_delays self.trace_length_stage2 = 2**(int(np.log2(self.pulse_length)) + 1) self.stage_2_window = half_hann_window(self.pulse_length, self.hann_window_fraction) self.stage_2_imager = II_tools.image_data_stage2_absBefore( S2_ant_locs, S2_ant_delays, self.trace_length_stage2, self.upsample_factor) self.erasure_window = 1.0 - self.stage_2_window