def get_trace_fromLoc(self, XYZT, ant_name, width, do_remove_RFI=True, do_remove_saturation=True, positive_saturation=2046, negative_saturation=-2047, removal_length=50, half_hann_length=50): """given the location of the source in XYZT, name of the antenna, and width (in num data samples) of the desired pulse, return starting_index of the returned trace, total time offest of that antenna, predicted arrival time of pulse at that antenna, and the time trace centered on the arrival time.""" station_name = SId_to_Sname[int(ant_name[:3])] station_data = self.data_file_dict[station_name] data_filter = self.data_filter_dict[station_name] file_antenna_index = station_data.get_antenna_names().index(ant_name) # ant_loc = station_data.get_LOFAR_centered_positions()[ file_antenna_index ] # ant_time_offset = station_data.get_timing_callibration_delays()[file_antenna_index] - station_data.get_nominal_sample_number()*5.0E-9 # total_time_offset = ant_time_offset + self.station_timing_calibration[station_name] # predicted_arrival_time = np.linalg.norm( XYZT[:3]-ant_loc )/v_air + XYZT[3] total_time_offset = station_data.get_total_delays()[file_antenna_index] antenna_locations = station_data.get_LOFAR_centered_positions() predicted_arrival_time = station_data.get_geometric_delays( XYZT[:3], antenna_locations=antenna_locations[ file_antenna_index:file_antenna_index + 1]) + XYZT[3] data_arrival_index = int(predicted_arrival_time / 5.0E-9 + total_time_offset / 5.0E-9) local_data_index = int(data_filter.blocksize * 0.5) data_start_sample = data_arrival_index - local_data_index input_data = station_data.get_data(data_start_sample, data_filter.blocksize, antenna_index=file_antenna_index) input_data = np.array(input_data, dtype=np.double) if do_remove_saturation: remove_saturation(input_data, positive_saturation, negative_saturation, removal_length, half_hann_length) width_before = int(width * 0.5) width_after = width - width_before if do_remove_RFI: data = data_filter.filter( input_data)[local_data_index - width_before:local_data_index + width_after] else: data = input_data[local_data_index - width_before:local_data_index + width_after] return data_start_sample + local_data_index - width_before, total_time_offset, predicted_arrival_time, data
def get_trace_fromIndex(self, starting_index, ant_name, width, do_remove_RFI=True, do_remove_saturation=True, positive_saturation=2046, negative_saturation=-2047, removal_length=50, saturation_half_hann_length=50): """similar to previous, but now retrieve trace based on location in file. For repeatability. Has same returns. predicted arrival time is just the time in middle of trace""" station_name = SId_to_Sname[int(ant_name[:3])] station_data = self.data_file_dict[station_name] data_filter = self.data_filter_dict[station_name] file_antenna_index = station_data.get_antenna_names().index(ant_name) total_time_offset = station_data.get_total_delays()[file_antenna_index] local_data_index = int(data_filter.blocksize * 0.5) input_data = station_data.get_data(starting_index - local_data_index, data_filter.blocksize, antenna_index=file_antenna_index) input_data = np.array(input_data, dtype=np.double) if do_remove_saturation: remove_saturation(input_data, positive_saturation, negative_saturation, removal_length, saturation_half_hann_length) if self.return_dbl_zeros: dbl_zeros = num_double_zeros( input_data[local_data_index:local_data_index + width]) if do_remove_RFI: data = data_filter.filter( input_data)[local_data_index:local_data_index + width] else: data = input_data[local_data_index:local_data_index + width] predicted_arrival_time = starting_index * 5.0E-9 - total_time_offset + 0.5 * width * 5.0E-9 if self.return_dbl_zeros: return starting_index, total_time_offset, predicted_arrival_time, data, dbl_zeros else: return starting_index, total_time_offset, predicted_arrival_time, data
def open_antenna_SampleNumber(self, antenna_i, sample_number): """opens the data for a station, where every antenna starts at the given sample number. Returns the data block for this station, and the start times for each antenna""" # print('opening:', antenna_i) station_i = self.antI_to_station[ antenna_i ] stationFile = self.station_data_files[ station_i ] data_filter = self.RFI_filters[ station_i ] station_antenna_index = self.station_antenna_index[ antenna_i ] TMP = stationFile.get_data(sample_number, self.blocksize, antenna_index=station_antenna_index ) if len(TMP) != self.blocksize: MSG = 'data length wrong. is: ' + str(len(TMP)) + " should be " + str(self.blocksize) + ". sample: " + \ str(sample_number) + ". antenna: " + str(station_antenna_index) + ". station: " + stationFile.get_station_name() raise Exception( MSG ) self.data_loss_spans[ antenna_i ], DL = locate_data_loss(TMP, self.num_dataLoss_zeros) self.tmp_workspace[:] = TMP if self.remove_saturation: self.staturation_removals[antenna_i] = remove_saturation( self.tmp_workspace, self.positive_saturation, self.negative_saturation, self.saturation_removal_length, self.saturation_half_hann_length ) else: self.staturation_removals[antenna_i] = [] if self.remove_RFI: self.raw_data[ antenna_i ] = data_filter.filter( self.tmp_workspace ) else: self.raw_data[ antenna_i ] = self.tmp_workspace self.starting_time[ antenna_i ] = sample_number*5E-9 - self.antenna_delays[ antenna_i ] self.antenna_data_loaded[ antenna_i ] = True return self.raw_data[ antenna_i ], self.starting_time[ antenna_i ]
bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) RFI_filter = window_and_filter(timeID=timeID, sname=station) data = np.empty(block_size, dtype=np.double) ant_names = TBB_data.get_antenna_names() num_antenna_pairs = int(len(ant_names) / 2) t0 = np.arange(block_size) H = 0 T = t0 + point for pair in range(num_antenna_pairs): data[:] = TBB_data.get_data(point, block_size, antenna_index=pair * 2) remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) filtered_data = RFI_filter.filter(data) even_HE = np.abs(filtered_data) data[:] = TBB_data.get_data(point, block_size, antenna_index=pair * 2 + 1) remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) filtered_data = RFI_filter.filter(data) odd_HE = np.abs(filtered_data) peak = max(np.max(even_HE), np.max(odd_HE))
def process_block(self, start_index, block_index, h5_groupobject, log_func=do_nothing): #### open and filter the data #### prefered_ant_i = None prefered_ant_dataLoss = np.inf for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use): data_file = self.input_files[station_i] RFI_filter = self.RFI_filters[station_i] offset = self.antenna_data_offsets[ant_i] self.data_block[ant_i, :] = data_file.get_data(start_index+offset, self.block_size, antenna_index=station_ant_i) ## get the data. accounting for the offsets calculated earlier remove_saturation(self.data_block[ant_i, :], self.positive_saturation, self.negative_saturation, post_removal_length=self.saturation_removal_length, half_hann_length=self.saturation_hann_window_length) if ant_i in self.station_to_antenna_indeces_dict[ self.prefered_station ]: ## this must be done before filtering num_D_zeros = num_double_zeros( self.data_block[ant_i, :] ) if num_D_zeros < prefered_ant_dataLoss: ## this antenna could be teh antenna, in the prefered station, with least data loss prefered_ant_dataLoss = num_D_zeros prefered_ant_i = ant_i self.data_block[ant_i, :] = RFI_filter.filter( self.data_block[ant_i, :] ) ### make output object ### h5_groupobject = h5_groupobject.create_group(str(block_index)) h5_groupobject.attrs['block_index'] = block_index h5_groupobject.attrs['start_index'] = start_index h5_groupobject.attrs['prefered_ant_i'] = prefered_ant_i #### find sources #### np.abs( self.data_block[ prefered_ant_i ], out=self.hilbert_envelope_tmp ) for event_i in range(self.max_events_perBlock): ## find peak ## peak_loc = np.argmax(self.hilbert_envelope_tmp[self.startBlock_exclusion : -self.endBlock_exclusion ]) + self.startBlock_exclusion trace_start_loc = peak_loc - int( self.pulse_length/2 ) # pobably account for fact that trace is now longer than pulse_length on prefered antenna if self.hilbert_envelope_tmp[peak_loc] < self.min_pref_ant_amplitude: log_func("peaks are too small. Done searching") break ## select data for stage one ## s1_ant_i = 0 for ant_i, (station_i, station_ant_i) in enumerate(self.antennas_to_use): if self.use_core_stations_S1 or self.is_not_core[ant_i]: half_window_length = self.half_antenna_data_length[ant_i] self.stage_1_imager.set_data( self.data_block[ant_i][trace_start_loc : trace_start_loc+2*half_window_length], s1_ant_i ) s1_ant_i += 1 ## fft and xcorrelation ## self.stage_1_imager.prepare_image( prefered_ant_i, self.min_pulse_amplitude ) log_func("source:", event_i) stage_1_result, num_itter, num_stage1_itters = stochastic_minimizer(self.stage_1_imager.intensity, self.bounding_box, converg_num=self.stage_1_converg_num, converg_rad=self.stage_1_converg_radius, max_itters=self.stage_1_max_itters) log_func(" stoch. itters:", num_itter, num_stage1_itters) ## select data for stage 2 ## previous_solution = stage_1_result converged = False for stage2loop_i in range(self.stage_2_max_itters): problem = False s2_ant_i = -1 for ant_i in range( self.num_antennas ): if self.use_core_stations_S2 or self.is_not_core[ant_i]: s2_ant_i += 1 ## do this here cause of break below modeled_dt = -( np.linalg.norm( self.antenna_locations[ prefered_ant_i ]-previous_solution.x ) - np.linalg.norm( self.antenna_locations[ant_i]-previous_solution.x ) )/v_air modeled_dt -= self.antenna_delays[ prefered_ant_i ] - self.antenna_delays[ant_i] modeled_dt /= 5.0E-9 modeled_dt += peak_loc modeled_dt = int(modeled_dt) if modeled_dt+int(self.pulse_length/2) >= len(self.data_block[ant_i]): problem = True break self.stage_2_imager.set_data( self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)]*self.stage_2_window, s2_ant_i, -modeled_dt*5.0E-9 ) if problem: log_func("unknown problem. LOC at", previous_solution.x) self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] = 0.0 continue ## fft and and xcorrelation ## self.stage_2_imager.prepare_image( self.min_pulse_amplitude ) # BB = np.array( [ [previous_solution.x[0]-50, previous_solution.x[0]+50], [previous_solution.x[1]-50, previous_solution.x[1]+50], [previous_solution.x[2]-50, previous_solution.x[2]+50] ] ) stage_2_result, s2_itternum, num_stage2_itters = stochastic_minimizer(self.stage_2_imager.intensity_ABSbefore , BB, converg_num=5, test_spot=previous_solution.x, converg_rad=self.stage_2_convergence_length, max_itters=self.stage_2_max_stoch_itters, options={'maxiter':1000}) D = np.linalg.norm( stage_2_result.x - previous_solution.x ) log_func(" s2 itter: {:2d} {:4.2f} {:d}".format(stage2loop_i, -stage_2_result.fun, int(D)) ) if D < self.stage_2_convergence_length: converged = True break elif D > self.stage_2_break_length: converged = False break previous_solution = stage_2_result new_stage_1_result = minimize(self.stage_1_imager.intensity, stage_2_result.x, method="Nelder-Mead", options={'maxiter':1000}) log_func(" old S1: {:4.2f} new SH1: {:4.2f}".format(-stage_1_result.fun, -new_stage_1_result.fun) ) if stage_1_result.fun < new_stage_1_result.fun: S1_S2_distance = np.linalg.norm(stage_1_result.x-stage_2_result.x) else: S1_S2_distance = np.linalg.norm(new_stage_1_result.x-stage_2_result.x) log_func(" loc: {:d} {:d} {:d}".format(int(stage_2_result.x[0]), int(stage_2_result.x[1]), int(stage_2_result.x[2])) ) log_func(" S1-S2 distance: {:d} converged: {} ".format( int(S1_S2_distance), converged) ) log_func(" intensity: {:4.2f} amplitude: {:d} ".format( -stage_2_result.fun, int(self.hilbert_envelope_tmp[peak_loc])) ) log_func() log_func() ## save to file ## source_dataset = h5_groupobject.create_dataset(str(event_i), (self.num_antennas,self.pulse_length), dtype=np.complex) source_dataset.attrs["loc"] = stage_2_result.x source_dataset.attrs["unique_index"] = block_index*self.max_events_perBlock + event_i source_time_s2 = (peak_loc+start_index+self.antenna_data_offsets[prefered_ant_i])*5.0E-9 - np.linalg.norm( stage_2_result.x - self.antenna_locations[ prefered_ant_i ] )/v_air source_time_s2 -= self.prefered_station_timing_offset + self.prefered_station_antenna_timing_offsets[ self.antennas_to_use[prefered_ant_i][1] ] source_dataset.attrs["T"] = source_time_s2 source_dataset.attrs["peak_index"] = peak_loc source_dataset.attrs["intensity"] = -stage_2_result.fun source_dataset.attrs["stage_1_success"] = (num_stage1_itters==self.stage_1_converg_num) source_dataset.attrs["stage_1_num_itters"] = num_stage1_itters source_dataset.attrs["amplitude"] = self.hilbert_envelope_tmp[peak_loc] source_dataset.attrs["S1_S2_distance"] = S1_S2_distance source_dataset.attrs["converged"] = converged source_time_s1 = (peak_loc+start_index+self.antenna_data_offsets[prefered_ant_i])*5.0E-9 - np.linalg.norm( stage_1_result.x - self.antenna_locations[ prefered_ant_i ] )/v_air source_time_s1 -= self.prefered_station_timing_offset + self.prefered_station_antenna_timing_offsets[ self.antennas_to_use[prefered_ant_i][1] ] source_dataset.attrs["XYZT_s1"] = np.append(stage_1_result.x, [source_time_s1]) #### erase the peaks !! #### # self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] *= self.erasure_window self.hilbert_envelope_tmp[peak_loc-int(self.pulse_length/2):peak_loc+int(self.pulse_length/2)] = 0.0 if converged and self.erase_pulses: for ant_i in range( self.num_antennas ): modeled_dt = -( np.linalg.norm( self.antenna_locations[ prefered_ant_i ]-stage_2_result.x ) - np.linalg.norm( self.antenna_locations[ant_i]-stage_2_result.x ) )/v_air modeled_dt -= self.antenna_delays[ prefered_ant_i ] - self.antenna_delays[ant_i] modeled_dt /= 5.0E-9 modeled_dt += peak_loc modeled_dt = int(modeled_dt) source_dataset[ant_i] = self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)] self.data_block[ant_i, modeled_dt-int(self.pulse_length/2):modeled_dt+int(self.pulse_length/2)] *= self.erasure_window
def plot_blocks(timeID, block_size, block_starts, guess_delays, guess_location=None, bad_stations=[], polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays="ant_delays.txt", do_remove_saturation=True, do_remove_RFI=True, positive_saturation=2046, negative_saturation=-2047, saturation_post_removal_length=50, saturation_half_hann_length=5, referance_station="CS002"): """plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays, if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero. A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations""" if referance_station in guess_delays: ref_delay = guess_delays[referance_station] guess_delays = { sname: delay - ref_delay for sname, delay in guess_delays.items() } processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips) bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas) additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' + additional_antenna_delays) raw_fpaths = filePaths_by_stationName(timeID) raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \ for sname in raw_fpaths.keys() if sname not in bad_stations} if guess_location is not None: guess_location = np.array(guess_location) ref_stat_file = raw_data_files[referance_station] ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0] ref_delay = np.linalg.norm( ant_loc - guess_location[:3] ) / v_air - ref_stat_file.get_nominal_sample_number() * 5.0E-9 for sname, data_file in raw_data_files.items(): if sname not in guess_delays: guess_delays[sname] = 0.0 data_file = raw_data_files[sname] ant_loc = data_file.get_LOFAR_centered_positions()[0] guess_delays[sname] += ( np.linalg.norm(ant_loc - guess_location[:3]) / v_air - ref_delay) guess_delays[sname] -= data_file.get_nominal_sample_number( ) * 5.0E-9 RFI_filters = { sname: window_and_filter(timeID=timeID, sname=sname) for sname in raw_fpaths.keys() if sname not in bad_stations } data = np.empty(block_size, dtype=np.double) height = 0 t0 = np.arange(block_size) * 5.0E-9 transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData) sname_X_loc = 0.0 for sname, data_file in raw_data_files.items(): print(sname) station_delay = -data_file.get_nominal_sample_number() * 5.0E-9 if sname in guess_delays: station_delay = guess_delays[sname] station_delay_points = int(station_delay / 5.0E-9) RFI_filter = RFI_filters[sname] ant_names = data_file.get_antenna_names() num_antenna_pairs = int(len(ant_names) / 2) peak_height = 0.0 for point in block_starts: T = t0 + point * 5.0E-9 for pair in range(num_antenna_pairs): data[:] = data_file.get_data(point + station_delay_points, block_size, antenna_index=pair * 2) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter(data) else: filtered_data = hilbert(data) even_HE = np.abs(filtered_data) data[:] = data_file.get_data(point + station_delay_points, block_size, antenna_index=pair * 2 + 1) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter(data) else: filtered_data = hilbert(data) odd_HE = np.abs(filtered_data) plt.plot(T, even_HE + height, 'r') plt.plot(T, odd_HE + height, 'g') max_even = np.max(even_HE) if max_even > peak_height: peak_height = max_even max_odd = np.max(odd_HE) if max_odd > peak_height: peak_height = max_odd # plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height)) plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform) height += 2 * peak_height plt.show()
def plot_a_station(self, sname, height): print("plotting", sname) station_clock_offset = self.clock_offset_manager.get_visual_offsets( )[sname] antenna_delays = self.antenna_time_corrections[sname] TBB_file = self.TBB_files[sname] RFI_filter = self.RFI_filters[sname] initial_block = self.frame_manager.get_block() #### open data, track saturation #### saturated_ranges = {block_i: [] for block_i in range(self.num_blocks)} station_max = 0.0 for even_ant_i in range(0, len(antenna_delays), 2): delay = (antenna_delays[even_ant_i] + antenna_delays[even_ant_i + 1]) * 0.5 total_delay = delay + station_clock_offset delay_points = int(total_delay / 5.0E-9) for block_i in range(self.num_blocks): block_start = (block_i + initial_block) * self.block_size ##### even #### self.temp_data_block[even_ant_i, block_i, :] = TBB_file.get_data( block_start + delay_points, self.block_size, antenna_index=even_ant_i) if self.do_remove_saturation: sat_ranges = remove_saturation( self.temp_data_block[even_ant_i, block_i, :], self.positive_saturation, self.negative_saturation, self.saturation_post_removal_length, self.saturation_half_hann_length) saturated_ranges[block_i] += sat_ranges if self.do_remove_RFI: filtered_data = RFI_filter.filter( self.temp_data_block[even_ant_i, block_i, :]) else: filtered_data = hilbert(self.temp_data_block[even_ant_i, block_i, :]) self.temp_data_block[even_ant_i, block_i, :] = np.abs(filtered_data) peak = np.max(self.temp_data_block[even_ant_i, block_i, :]) if peak > station_max: station_max = peak ##### odd #### self.temp_data_block[even_ant_i + 1, block_i, :] = TBB_file.get_data( block_start + delay_points, self.block_size, antenna_index=even_ant_i + 1) if self.do_remove_saturation: sat_ranges = remove_saturation( self.temp_data_block[even_ant_i + 1, block_i, :], self.positive_saturation, self.negative_saturation, self.saturation_post_removal_length, self.saturation_half_hann_length) saturated_ranges[block_i] += sat_ranges if self.do_remove_RFI: filtered_data = RFI_filter.filter( self.temp_data_block[even_ant_i + 1, block_i, :]) else: filtered_data = hilbert( self.temp_data_block[even_ant_i + 1, block_i, :]) self.temp_data_block[even_ant_i + 1, block_i, :] = np.abs(filtered_data) peak = np.max(self.temp_data_block[even_ant_i + 1, block_i, :]) if peak > station_max: station_max = peak #### plot saturated bits #### time_by_block = { block_i: (self.time_array + (block_i + initial_block) * self.block_size * 5.0E-9) for block_i in range(self.num_blocks) } for block_i, ranges in saturated_ranges.items(): T = time_by_block[block_i] for imin, imax in ranges: Tmin = T[imin] width = T[imax - 1] - Tmin ## range does not include end point rect = patches.Rectangle((Tmin, height), width, 1, linewidth=1, edgecolor='r', facecolor='r') self.axes.add_patch(rect) #### plot data #### for pair_i in range(int(len(antenna_delays) / 2)): even_ant = pair_i * 2 odd_odd = pair_i * 2 + 1 for block_i in range(self.num_blocks): block_start = (block_i + initial_block) * self.block_size even_trace = self.temp_data_block[even_ant, block_i] odd_trace = self.temp_data_block[odd_odd, block_i] even_trace /= station_max odd_trace /= station_max even_trace += height odd_trace += height T = time_by_block[block_i] plt.plot(T, even_trace, 'g') plt.plot(T, odd_trace, 'm') minT, maxT = self.frame_manager.get_T_bounds() self.pulse_manager.plot_lines(sname, height, minT, maxT, station_clock_offset)
def __init__(self, TBB_data, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4, max_num_planewaves=np.inf, timeID=None, blocksize=2**16, positive_saturation=2046, negative_saturation=-2047, saturation_post_removal_length=50, saturation_half_hann_length=50, verbose=True): self.parabolic_fitter = parabolic_fitter() self.polarization = polarization self.TBB_data = TBB_data left_pulse_length = int(pulse_length / 2) right_pulse_length = pulse_length - left_pulse_length ant_names = TBB_data.get_antenna_names() num_antenna_pairs = int(len(ant_names) / 2) self.num_found_planewaves = 0 if num_antenna_pairs < min_num_antennas: return RFI_filter = window_and_filter( timeID=timeID, sname=TBB_data.get_station_name(), blocksize=(blocksize if timeID is None else None)) block_size = RFI_filter.blocksize self.num_found_planewaves = 0 data = np.empty((num_antenna_pairs, block_size), dtype=np.double) self.planewave_data = [] self.planewave_second_derivatives = [] for block in range(initial_block, initial_block + number_of_blocks): if self.num_found_planewaves >= max_num_planewaves: break #### open and filter data # print('open block', block) for pair_i in range(num_antenna_pairs): ant_i = pair_i * 2 + polarization data[pair_i, :] = TBB_data.get_data(block * block_size, block_size, antenna_index=ant_i) remove_saturation(data[pair_i, :], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) np.abs(RFI_filter.filter(data[pair_i, :]), out=data[pair_i, :]) #### loop over finding planewaves i = 0 while i < pulses_per_block: if self.num_found_planewaves >= max_num_planewaves: break pulse_location = 0 pulse_amplitude = 0 ## find highest peak for pair_i, HE in enumerate(data): loc = np.argmax(HE) amp = HE[loc] if amp > pulse_amplitude: pulse_amplitude = amp pulse_location = loc ## check if is strong enough if pulse_amplitude < min_amplitude: break ## get newPW_data = np.full(num_antenna_pairs, np.nan, dtype=np.double) newPW_derivatives = np.full(num_antenna_pairs, np.nan, dtype=np.double) for pair_i, HE in enumerate(data): # ant_i = pair_i*2 + polarization signal = np.array( HE[pulse_location - left_pulse_length:pulse_location + right_pulse_length]) if num_double_zeros(signal, threshold=0.1) == 0: sample_time = 5.0E-9 if upsample_factor > 1: sample_time /= upsample_factor signal = resample(signal, len(signal) * upsample_factor) newPW_data[pair_i] = self.parabolic_fitter.fit( signal) * sample_time newPW_derivatives[ pair_i] = self.parabolic_fitter.second_derivative( ) * sample_time HE[pulse_location - left_pulse_length:pulse_location + right_pulse_length] = 0.0 if np.sum(np.isfinite(newPW_data)) >= min_num_antennas: self.planewave_data.append(newPW_data) self.planewave_second_derivatives.append(newPW_derivatives) self.num_found_planewaves += 1 i += 1
def more_plots(some_data,First,ax,text): global l,m ax.clear() height = 0 t0 = np.arange(block_size)*5.0E-9 transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData) sname_X_loc = 0.0 n = 0 for sname, data_file in some_data: print("Plotting: "+sname) station_delay = -data_file.get_nominal_sample_number()*5.0E-9 if sname in guess_delays: station_delay = guess_delays[sname] station_delay_points = int(station_delay/5.0E-9) RFI_filter = RFI_filters[sname] ant_names = data_file.get_antenna_names() num_antenna_pairs = int( len( ant_names )/2 ) peak_height = 0.0 print("Block starts at: "+str(block_starts[0])) for point in block_starts: T = t0 + point*5.0E-9 for pair in range(num_antenna_pairs): data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter( data ) else: filtered_data = hilbert(data) even_HE = np.abs(filtered_data) data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2+1) if do_remove_saturation: remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) if do_remove_RFI: filtered_data = RFI_filter.filter( data ) else: filtered_data = hilbert(data) odd_HE = np.abs(filtered_data) #ax.plot(T, even_HE + height, 'r') #ax.plot(T, odd_HE + height, 'g' ) #for cases where data are hard to align due to signal being tiny: found = False if len(amplify) > 0: for key in amplify: if sname == key: if omitPOL != 0: ax.plot(T, amplify[key]*even_HE + height, 'r') if omitPOL != 1: ax.plot(T, amplify[key]*odd_HE + height, 'g' ) #ax.plot(T, amplify[key]*even_HE + height, 'r') twinx with block_starts && point as x rather than t! (can I update to latests plot_multiple?) #ax.plot(T, amplify[key]*odd_HE + height, 'g' ) found = True if not found: if omitPOL != 0: ax.plot(T, even_HE + height, 'r') if omitPOL != 1: ax.plot(T, odd_HE + height, 'g' ) #ax2 = ax.twiny() #xmin = min(block_starts) #xmax = max(block_starts) #ax2.set_xlim(xmin, xmax) #can also use ax2.set_xticks(x) #ax2.xaxis.set_ticks_position('both') max_even = np.max(even_HE) if max_even > peak_height: peak_height = max_even max_odd = np.max(odd_HE) if max_odd > peak_height: peak_height = max_odd # plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height)) plt.sca(ax) #somehow this makes it so that the annotations work on all plots except for the first time "Next stations" is clicked plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform) ax.ticklabel_format(useOffset=False) plt.setp(ax.get_xticklabels(), rotation=60, horizontalalignment='right') plt.subplots_adjust(bottom=0.1,top=0.98,left=0.21) if ref_plot_peak_location != 0.0: print("Setting ref peak to " +str(ref_plot_peak_location)) Cpeak = ref_plot_peak_location #peak center, for plotting Dpeak = 0.0001 # +/- added to peak center for plotting plt.xlim(Cpeak-Dpeak,Cpeak+Dpeak) plt.grid(b=True) height += 2*peak_height n+=1
def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4, polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", max_num_planewaves=np.inf, positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 50, verbose=True): left_pulse_length = int(pulse_length/2) right_pulse_length = pulse_length-left_pulse_length processed_data_folder = processed_data_dir(timeID) polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips ) bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas ) additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays ) raw_fpaths = filePaths_by_stationName(timeID) TBB_data = MultiFile_Dal1( raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays ) ant_names = TBB_data.get_antenna_names() antenna_locations = TBB_data.get_LOFAR_centered_positions() antenna_delays = TBB_data.get_timing_callibration_delays() num_antenna_pairs = int( len( ant_names )/2 ) if num_antenna_pairs < min_num_antennas: return np.array([]), np.array([]), np.array([]) RFI_filter = window_and_filter(timeID=timeID, sname=station) block_size = RFI_filter.blocksize out_RMS = np.empty( number_of_blocks*pulses_per_block, dtype=np.double ) out_zenith = np.empty( number_of_blocks*pulses_per_block, dtype=np.double ) out_azimuth = np.empty( number_of_blocks*pulses_per_block, dtype=np.double ) N = 0 data = np.empty( (num_antenna_pairs,block_size), dtype=np.double ) for block in range(initial_block, initial_block+number_of_blocks): if N >= max_num_planewaves: break #### open and filter data for pair_i in range(num_antenna_pairs): ant_i = pair_i*2 + polarization data[pair_i,:] = TBB_data.get_data(block*block_size, block_size, antenna_index=ant_i) remove_saturation(data[pair_i,:], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length) np.abs( RFI_filter.filter( data[pair_i,:] ), out=data[pair_i,:]) #### loop over finding planewaves i = 0 while i < pulses_per_block: if N >= max_num_planewaves: break pulse_location = 0 pulse_amplitude = 0 ## find highest peak for pair_i, HE in enumerate(data): loc = np.argmax( HE ) amp = HE[ loc ] if amp > pulse_amplitude: pulse_amplitude = amp pulse_location = loc ## check if is strong enough if pulse_amplitude < min_amplitude: break ## get fitter = locatefier() for pair_i, HE in enumerate(data): ant_i = pair_i*2 + polarization signal = np.array( HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] ) if num_double_zeros(signal, threshold=0.1) == 0: sample_time = 5.0E-9 if upsample_factor > 1: sample_time /= upsample_factor signal = resample(signal, len(signal)*upsample_factor ) peak_finder = parabolic_fit( signal ) peak_time = peak_finder.peak_index*sample_time - antenna_delays[ant_i] fitter.add_antenna(peak_time, antenna_locations[ant_i]) HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] = 0.0 if fitter.num_measurments() < min_num_antennas: continue fitter.prep_for_fitting() X = brute(fitter.RMS, ranges=[[0,np.pi/2],[0,np.pi*2]], Ns=20) if X[0] >= np.pi/2: X[0] = (np.pi/2)*0.99 if X[1] >= np.pi*2: X[1] = (np.pi*2)*0.99 if X[0] < 0: X[0] = 0.00000001 if X[1] < 0: X[1] = 0.00000001 ret = least_squares( fitter.time_fits, X, bounds=[[0,0],[np.pi/2,2*np.pi]], ftol=3e-16, xtol=3e-16, gtol=3e-16, x_scale='jac' ) out_RMS[N] = fitter.RMS(ret.x, 3) out_zenith[N] = ret.x[0] out_azimuth[N] = ret.x[1] N += 1 i += 1 return out_RMS[:N], out_zenith[:N], out_azimuth[:N]