def __init__(self, timeID, pulseID, alt_loc=None):
self.data_loc = processed_data_dir(timeID) + '/' + "polarization_data"
if alt_loc is not None:
self.data_loc = alt_loc
#if the directory doesn't exist create it!
if not isdir(self.data_loc):
mkdir(self.data_loc)
if not isdir(self.data_loc + '/' + "Stokes_Vectors"):
mkdir(self.data_loc + '/' + "Stokes_Vectors")
if not isdir(self.data_loc + '/' + "Polarization_Ellipse"):
mkdir(self.data_loc + '/' + "Polarization_Ellipse")
self.filename_S = "{}.txt".format(pulseID) #filename for stokesvectors
self.filename_PE = "{}.txt".format(
pulseID) #filename for polarization ellipse data
#create empty files with header for S and PE for a particular pulse
with open(self.data_loc + "/Stokes_Vectors/" + self.filename_S,
'w') as f:
f.write("# Station I Q U V Ierr Qerr Uerr Verr\n")
with open(self.data_loc + "/Polarization_Ellipse/" + self.filename_PE,
'w') as f:
f.write("# Station S_0 δ τ ε σ_S_0 σ_δ σ_τ σ_ε\n")
def __init__(self,
load_gal_cal=True,
timeID=None,
findRFI=None,
cal_curve=get_LBA_frequency_calibration,
jones_matrices=None):
"""if load_gal_cal is true, then timeID and findRFI should indicate RFI info. If load_gal_cal is false, than data is not normalized relative to noise.
cal_curve should take numpy arrays of frequency and return frequency dependant adjustment of the data. can be None.
Jones matrices should be a function that has three parameters: 1) numpy array of frequencies in Hz, 2) zenith in degrees, and 3) azimuth in degrees, then returns array of jones matrices. Default is pycrtools model """
self.cal_curve = cal_curve
if load_gal_cal:
if findRFI is None:
findRFI = "/findRFI/findRFI_results"
if isinstance(findRFI, str): ## load findRFI data from file
galcal_data_loc = processed_data_dir(timeID) + findRFI
with open(galcal_data_loc, 'rb') as fin:
findRFI = load(fin)
self.calibration_factors = {}
for findRFI_info in findRFI.values():
antenna_names = findRFI_info["antenna_names"]
num_antennas = len(antenna_names)
cleaned_power = findRFI_info["cleaned_power"]
timestamp = findRFI_info["timestamp"]
analyzed_blocksize = findRFI_info["blocksize"]
even_cal_factors, odd_cal_factors = getGalaxyCalibrationData(
cleaned_power, timestamp, 5.0E-9 / analyzed_blocksize)
for ant_i in range(0, int(num_antennas / 2)):
ant_name = antenna_names[ant_i * 2]
self.calibration_factors[ant_name] = (
even_cal_factors[ant_i], odd_cal_factors[ant_i])
else:
self.calibration_factors = None
### Galaxy callibration data ###
# self.calibration_factors = {}
# galcal_data_loc = processed_data_dir(timeID) + '/cal_tables/galaxy_cal'
# galcal_fpaths = glob.glob(galcal_data_loc + '/*.gcal')
# for fpath in galcal_fpaths:
# with open(fpath, 'rb') as fin:
# data = np.load(fin)
# ant_names = data["arr_0"].astype(str, copy=False)
# factors = data["arr_1"]
# ant_i = 0
# while ant_i<len(ant_names):
# self.calibration_factors[ ant_names[ant_i] ] = [factors[ant_i], factors[ant_i+1]]
# ant_i += 2
if jones_matrices is None:
self.antenna_model = LBA_antenna_model().JonesMatrix_MultiFreq
def planewave_fits(timeID,
station,
polarization,
initial_block,
number_of_blocks,
pulses_per_block=10,
pulse_length=50,
min_amplitude=50,
upsample_factor=0,
min_num_antennas=4,
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt",
max_num_planewaves=np.inf,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
verbose=True):
""" wrapper around 'planewave_fitter' only present for backwards compatiblility. Returns RMS, zenithal and azimuthal fit values, as three arrays in that order"""
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder +
'/' +
additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = MultiFile_Dal1(raw_fpaths[station],
polarization_flips=polarization_flips,
bad_antennas=bad_antennas,
additional_ant_delays=additional_antenna_delays)
fitter = planewave_fitter(
TBB_data=TBB_data,
polarization=polarization,
initial_block=initial_block,
number_of_blocks=number_of_blocks,
pulses_per_block=pulses_per_block,
pulse_length=pulse_length,
min_amplitude=min_amplitude,
upsample_factor=upsample_factor,
min_num_antennas=min_num_antennas,
max_num_planewaves=max_num_planewaves,
verbose=verbose, ## doesn't do anything anyway
positive_saturation=positive_saturation,
negative_saturation=negative_saturation,
saturation_post_removal_length=saturation_post_removal_length,
saturation_half_hann_length=saturation_half_hann_length)
RMSs, Zeniths, Azimuths, throw = fitter.go_fit()
return RMSs, Zeniths, Azimuths
def __init__(self, timeID, alt_loc=None):
self.data_loc = processed_data_dir(timeID)
if alt_loc is not None:
self.data_loc = alt_loc
else:
if not isdir(self.data_loc + '/' + "polarization_data"):
raise FileNotFoundError(
errno.ENOENT, strerror(errno.ENOENT),
"{}".format(self.data_loc + '/' + "polarization_data"))
self.data_loc += "/polarization_data"
def __init__(self, timeID, fname=None, alt_loc=None):
self.data_loc = processed_data_dir(timeID)
if alt_loc is not None:
self.data_loc = alt_loc
else:
if not isdir(self.data_loc + '/' + "polarization_data"):
raise FileNotFoundError(
errno.ENOENT, strerror(errno.ENOENT),
"{}".format(self.data_loc + '/' + "polarization_data"))
self.data_loc += "/polarization_data"
if fname is not None:
self.save_file = fname + ".json"
else:
self.save_file = "a_vectors.json"
#create the file
with open(self.data_loc + '/' + self.save_file, 'w') as f:
json.dump(dict(), f, indent=4)
def __init__(self, timeID=None, findRFI=None):
if findRFI is None:
findRFI = "/findRFI/findRFI_results"
if isinstance(findRFI, str): ## load findRFI data from file
galcal_data_loc = processed_data_dir(timeID) + findRFI
with open(galcal_data_loc, 'rb') as fin:
findRFI = load(fin)
self.calibration_factors = {}
for findRFI_info in findRFI.values():
antenna_names = findRFI_info["antenna_names"]
num_antennas = len(antenna_names)
cleaned_power = findRFI_info["cleaned_power"]
timestamp = findRFI_info["timestamp"]
analyzed_blocksize = findRFI_info["blocksize"]
even_cal_factors, odd_cal_factors = getGalaxyCalibrationData(
cleaned_power, timestamp, 5.0E-9 / analyzed_blocksize)
for ant_i in range(0, int(num_antennas / 2)):
ant_name = antenna_names[ant_i * 2]
self.calibration_factors[ant_name] = (even_cal_factors[ant_i],
odd_cal_factors[ant_i])
### Galaxy callibration data ###
# self.calibration_factors = {}
# galcal_data_loc = processed_data_dir(timeID) + '/cal_tables/galaxy_cal'
# galcal_fpaths = glob.glob(galcal_data_loc + '/*.gcal')
# for fpath in galcal_fpaths:
# with open(fpath, 'rb') as fin:
# data = np.load(fin)
# ant_names = data["arr_0"].astype(str, copy=False)
# factors = data["arr_1"]
# ant_i = 0
# while ant_i<len(ant_names):
# self.calibration_factors[ ant_names[ant_i] ] = [factors[ant_i], factors[ant_i+1]]
# ant_i += 2
self.antenna_model = LBA_antenna_model()
def read_acceleration_vector(timeID, fname, alt_loc=None):
data_loc = processed_data_dir(timeID)
if alt_loc is not None:
data_loc = alt_loc
else:
if not isdir(data_loc):
raise FileNotFoundError(errno.ENOENT, strerror(errno.ENOENT),
"{}".format(data_loc))
data_loc += "/polarization_data"
if fname is not None:
save_file = fname + ".json"
else:
save_file = "a_vectors.json"
if not isfile(data_loc + '/' + save_file):
raise FileNotFoundError(errno.ENOENT, strerror(errno.ENOENT),
"{}".format(data_loc + '/' + save_file))
with open(data_loc + '/' + save_file, 'r') as f:
data = json.load(f)
return data
def get_noise_std(timeID,
initial_block,
max_num_blocks,
max_double_zeros=100,
stations=None,
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt"):
processed_data_folder = processed_data_dir(timeID)
if isinstance(polarization_flips, str):
polarization_flips = read_antenna_pol_flips(processed_data_folder +
'/' + polarization_flips)
if isinstance(processed_data_folder, str):
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
half_window_percent = 0.1
half_window_percent *= 1.1 ## just to be sure we are away from edge
raw_fpaths = filePaths_by_stationName(timeID)
if stations is None:
stations = raw_fpaths.keys()
out_dict = {}
for sname in stations:
print(sname)
TBB_data = MultiFile_Dal1(raw_fpaths[sname],
polarization_flips=polarization_flips,
bad_antennas=bad_antennas)
RFI_filter = window_and_filter(timeID=timeID, sname=sname)
block_size = RFI_filter.blocksize
edge_size = int(half_window_percent * block_size)
antenna_names = TBB_data.get_antenna_names()
measured_std = np.zeros(len(antenna_names))
measured_std[:] = -1
for block_i in range(initial_block, initial_block + max_num_blocks):
for ant_i in range(len(antenna_names)):
if measured_std[ant_i] > 0:
continue ## we got a measurment, so skip it
data = np.array(TBB_data.get_data(block_i * block_size,
block_size,
antenna_index=ant_i),
dtype=np.double)
if num_double_zeros(data) > max_double_zeros:
continue ## bad block, skip
filtered_data = np.real(RFI_filter.filter(data))
filtered_data = filtered_data[edge_size:
-edge_size] ##avoid the edge
measured_std[ant_i] = np.std(filtered_data)
if np.all(measured_std > 0): ## we can break early
break
for ant_name, std in zip(antenna_names, measured_std):
out_dict[ant_name] = std
filter = measured_std > 0
print(" ave std:", np.average(measured_std[filter]))
print(" total ant:", len(filter), "good ant:", np.sum(filter))
print()
return out_dict
def plot_blocks(timeID,
block_size,
block_starts,
guess_delays,
guess_location=None,
bad_stations=[],
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt",
do_remove_saturation=True,
do_remove_RFI=True,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=5,
referance_station="CS002"):
"""plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays,
if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero.
A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations"""
if referance_station in guess_delays:
ref_delay = guess_delays[referance_station]
guess_delays = {
sname: delay - ref_delay
for sname, delay in guess_delays.items()
}
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder +
'/' +
additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in raw_fpaths.keys() if sname not in bad_stations}
if guess_location is not None:
guess_location = np.array(guess_location)
ref_stat_file = raw_data_files[referance_station]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(
ant_loc - guess_location[:3]
) / v_air - ref_stat_file.get_nominal_sample_number() * 5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in guess_delays:
guess_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
guess_delays[sname] += (
np.linalg.norm(ant_loc - guess_location[:3]) / v_air -
ref_delay)
guess_delays[sname] -= data_file.get_nominal_sample_number(
) * 5.0E-9
RFI_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in raw_fpaths.keys() if sname not in bad_stations
}
data = np.empty(block_size, dtype=np.double)
height = 0
t0 = np.arange(block_size) * 5.0E-9
transform = blended_transform_factory(plt.gca().transAxes,
plt.gca().transData)
sname_X_loc = 0.0
for sname, data_file in raw_data_files.items():
print(sname)
station_delay = -data_file.get_nominal_sample_number() * 5.0E-9
if sname in guess_delays:
station_delay = guess_delays[sname]
station_delay_points = int(station_delay / 5.0E-9)
RFI_filter = RFI_filters[sname]
ant_names = data_file.get_antenna_names()
num_antenna_pairs = int(len(ant_names) / 2)
peak_height = 0.0
for point in block_starts:
T = t0 + point * 5.0E-9
for pair in range(num_antenna_pairs):
data[:] = data_file.get_data(point + station_delay_points,
block_size,
antenna_index=pair * 2)
if do_remove_saturation:
remove_saturation(data, positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter(data)
else:
filtered_data = hilbert(data)
even_HE = np.abs(filtered_data)
data[:] = data_file.get_data(point + station_delay_points,
block_size,
antenna_index=pair * 2 + 1)
if do_remove_saturation:
remove_saturation(data, positive_saturation,
negative_saturation,
saturation_post_removal_length,
saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter(data)
else:
filtered_data = hilbert(data)
odd_HE = np.abs(filtered_data)
plt.plot(T, even_HE + height, 'r')
plt.plot(T, odd_HE + height, 'g')
max_even = np.max(even_HE)
if max_even > peak_height:
peak_height = max_even
max_odd = np.max(odd_HE)
if max_odd > peak_height:
peak_height = max_odd
# plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
plt.annotate(sname, (sname_X_loc, height),
textcoords=transform,
xycoords=transform)
height += 2 * peak_height
plt.show()
def open_files(self, log_func=do_nothing, force_metadata_ant_pos=True):
processed_data_loc = processed_data_dir(self.timeID)
#### open callibration data files ####
### TODO: fix these so they are accounted for in a more standard maner
self.station_timing_offsets = read_station_delays(
processed_data_loc + '/' + self.station_delays_fname)
self.additional_ant_delays = read_antenna_delays(
processed_data_loc + '/' + self.additional_antenna_delays_fname)
self.bad_antennas = read_bad_antennas(processed_data_loc + '/' +
self.bad_antennas_fname)
self.pol_flips = read_antenna_pol_flips(processed_data_loc + '/' +
self.pol_flips_fname)
#### open data files, and find RFI ####
raw_fpaths = filePaths_by_stationName(self.timeID)
self.station_names = []
self.input_files = []
self.RFI_filters = []
CS002_index = None
self.station_to_antenna_indeces_dict = {}
for station, fpaths in raw_fpaths.items():
if (station in self.station_timing_offsets) and (
station not in self.stations_to_exclude) and (
self.use_core_stations_S1 or self.use_core_stations_S2
or (not station[:2] == 'CS') or station == "CS002"):
# if (station not in self.stations_to_exclude) and ( self.use_core_stations_S1 or self.use_core_stations_S2 or (not station[:2]=='CS') or station=="CS002" ):
log_func("opening", station)
self.station_names.append(station)
self.station_to_antenna_indeces_dict[station] = []
if station == 'CS002':
CS002_index = len(self.station_names) - 1
new_file = MultiFile_Dal1(
fpaths, force_metadata_ant_pos=force_metadata_ant_pos)
new_file.set_polarization_flips(self.pol_flips)
self.input_files.append(new_file)
RFI_result = None
if self.do_RFI_filtering and self.use_saved_RFI_info:
self.RFI_filters.append(
window_and_filter(timeID=self.timeID, sname=station))
elif self.do_RFI_filtering:
RFI_result = FindRFI(new_file,
self.block_size,
self.initial_RFI_block,
self.RFI_num_blocks,
self.RFI_max_blocks,
verbose=False,
figure_location=None)
self.RFI_filters.append(
window_and_filter(find_RFI=RFI_result))
else: ## only basic filtering
self.RFI_filters.append(
window_and_filter(blocksize=self.block_size))
#### find antenna pairs ####
boundingBox_center = np.average(self.bounding_box, axis=-1)
self.antennas_to_use = pairData_NumAntPerStat(
self.input_files, self.num_antennas_per_station, self.bad_antennas)
# self.antennas_to_use = pairData_NumAntPerStat_PolO(self.input_files, self.num_antennas_per_station, self.bad_antennas )
# self.antennas_to_use = pairData_NumAntPerStat_DualPol(self.input_files, self.num_antennas_per_station, self.bad_antennas )
self.num_antennas = len(self.antennas_to_use)
#### get antenna locations and delays ####
self.antenna_locations = np.zeros((self.num_antennas, 3),
dtype=np.double)
self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double)
self.is_not_core = np.zeros(self.num_antennas, dtype=np.bool)
#### TODO: Do we need this "CSOO2 correction??? will removing it fix our time base?? ####
# clock_corrections = getClockCorrections()
# CS002_correction = -clock_corrections["CS002"] - self.input_files[CS002_index].get_nominal_sample_number()*5.0E-9 ## wierd sign. But is just to correct for previous definiitions
CS002_correction = self.station_timing_offsets[
'CS002'] - self.input_files[CS002_index].get_nominal_sample_number(
) * 5.0E-9 ## wierd sign. But is just to correct for previous definiitions
for ant_i, (station_i,
station_ant_i) in enumerate(self.antennas_to_use):
self.station_to_antenna_indeces_dict[
self.station_names[station_i]].append(
ant_i
) ### TODO: this should probably be a result of pairData_NumAntPerStat
data_file = self.input_files[station_i]
station = self.station_names[station_i]
self.is_not_core[ant_i] = (not station[:2]
== 'CS') or station == "CS002"
ant_name = data_file.get_antenna_names()[station_ant_i]
self.antenna_locations[
ant_i] = data_file.get_LOFAR_centered_positions(
)[station_ant_i]
self.antenna_delays[
ant_i] = data_file.get_timing_callibration_delays(
)[station_ant_i]
## account for station timing offsets
# self.antenna_delays[ant_i] += self.station_timing_offsets[station] + (-clock_corrections[station] - data_file.get_nominal_sample_number()*5.0E-9) - CS002_correction
self.antenna_delays[ant_i] += self.station_timing_offsets[
station] + (-data_file.get_nominal_sample_number() *
5.0E-9) - CS002_correction
## add additional timing delays
##note this only works for even antennas!
if ant_name in self.additional_ant_delays:
self.antenna_delays[ant_i] += self.additional_ant_delays[
ant_name][0]
# #### find prefered station ####
if self.prefered_station is None:
prefered_stat_shortestWindowTime = np.inf
prefered_station_input_file = None
for station, antenna_indeces in self.station_to_antenna_indeces_dict.items(
):
ant_i = antenna_indeces[0] ## just use first antenna for test
if not (self.use_core_stations_S1 or self.is_not_core[ant_i]):
continue
longest_window_time = 0
for ant_j in range(self.num_antennas):
if not (self.use_core_stations_S1
or self.is_not_core[ant_j]):
continue
window_time, throw, throw = find_max_duration(
self.antenna_locations[ant_i],
self.antenna_locations[ant_j],
self.bounding_box,
center=boundingBox_center)
if window_time > longest_window_time:
longest_window_time = window_time
if longest_window_time < prefered_stat_shortestWindowTime:
prefered_stat_shortestWindowTime = longest_window_time
self.prefered_station = station
prefered_station_input_file = self.input_files[
self.antennas_to_use[ant_i][0]]
else: ### TODO: throw error is prefered station is in core, but core not included in stage 1
ant_i = self.station_to_antenna_indeces_dict[
self.prefered_station][0]
prefered_stat_shortestWindowTime = 0
for ant_j in range(self.num_antennas):
if not (self.use_core_stations_S1 or self.is_not_core[ant_j]):
continue
window_time, throw, throw = find_max_duration(
self.antenna_locations[ant_i],
self.antenna_locations[ant_j],
self.bounding_box,
center=boundingBox_center)
if window_time > prefered_stat_shortestWindowTime:
prefered_stat_shortestWindowTime = window_time
prefered_station_input_file = self.input_files[
self.antennas_to_use[ant_i][0]]
log_func("prefered station:", self.prefered_station)
log_func(" max. half window time:",
prefered_stat_shortestWindowTime)
log_func()
log_func()
self.prefered_station_timing_offset = self.station_timing_offsets[
self.
prefered_station] - prefered_station_input_file.get_nominal_sample_number(
) * 5.0E-9
self.prefered_station_antenna_timing_offsets = prefered_station_input_file.get_timing_callibration_delays(
)
self.startBlock_exclusion = int(
0.1 * self.block_size) ### TODO: save these to header.
self.endBlock_exclusion = int(
prefered_stat_shortestWindowTime / 5.0E-9) + 1 + int(
0.1 * self.block_size) ## last bit accounts for hann-window
self.active_block_length = self.block_size - self.startBlock_exclusion - self.endBlock_exclusion ##the length of block used for looking at data
##### find window offsets and lengths ####
self.antenna_data_offsets = np.zeros(self.num_antennas, dtype=np.long)
self.half_antenna_data_length = np.zeros(self.num_antennas,
dtype=np.long)
for ant_i, (station_i,
station_ant_i) in enumerate(self.antennas_to_use):
## now we account for distance to the source
travel_time = np.linalg.norm(self.antenna_locations[ant_i] -
boundingBox_center) / v_air
self.antenna_data_offsets[ant_i] = int(travel_time / 5.0E-9)
### find max duration to any of the prefered antennas
max_duration = 0.0
for prefered_ant_i in self.station_to_antenna_indeces_dict[
self.prefered_station]:
if prefered_ant_i == ant_i:
continue
duration, throw, throw = find_max_duration(
self.antenna_locations[prefered_ant_i],
self.antenna_locations[ant_i], self.bounding_box,
boundingBox_center)
if duration > max_duration:
max_duration = duration
self.half_antenna_data_length[ant_i] = int(
self.pulse_length / 2) + int(duration / 5.0E-9)
self.antenna_data_offsets[ant_i] -= self.half_antenna_data_length[
ant_i]
#### now adjust the data offsets and antenna delays so they are consistent
## first we amount to adjust the offset by time delays mod the sampling time
offset_adjust = int(
self.antenna_delays[ant_i] / 5.0E-9
) ##this needs to be added to offsets and subtracted from delays
## then we can adjust the delays accounting for the data offset
self.antenna_delays[
ant_i] -= self.antenna_data_offsets[ant_i] * 5.0E-9
##now we finally account for large time delays
self.antenna_data_offsets[ant_i] += offset_adjust
self.antenna_delays[ant_i] -= offset_adjust * 5.0E-9
if (not self.use_core_stations_S1) or (not self.use_core_stations_S2):
core_filtered_ant_locs = np.array(
self.antenna_locations[self.is_not_core])
core_filtered_ant_delays = np.array(
self.antenna_delays[self.is_not_core])
#### allocate some memory ####
self.data_block = np.empty((self.num_antennas, self.block_size),
dtype=np.complex)
self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double)
#### initialize stage 1 ####
if self.use_core_stations_S1:
self.trace_length_stage1 = 2 * np.max(
self.half_antenna_data_length)
S1_ant_locs = self.antenna_locations
S1_ant_delays = self.antenna_delays
else:
self.trace_length_stage1 = 2 * np.max(
self.half_antenna_data_length[self.is_not_core])
S1_ant_locs = core_filtered_ant_locs
S1_ant_delays = core_filtered_ant_delays
self.trace_length_stage1 = 2**(int(np.log2(self.trace_length_stage1)) +
1)
self.stage_1_imager = II_tools.image_data_stage1(
S1_ant_locs, S1_ant_delays, self.trace_length_stage1,
self.upsample_factor)
#### initialize stage 2 ####
if self.use_core_stations_S2:
S2_ant_locs = self.antenna_locations
S2_ant_delays = self.antenna_delays
else:
S2_ant_locs = core_filtered_ant_locs
S2_ant_delays = core_filtered_ant_delays
self.trace_length_stage2 = 2**(int(np.log2(self.pulse_length)) + 1)
self.stage_2_window = half_hann_window(self.pulse_length,
self.hann_window_fraction)
self.stage_2_imager = II_tools.image_data_stage2_absBefore(
S2_ant_locs, S2_ant_delays, self.trace_length_stage2,
self.upsample_factor)
self.erasure_window = 1.0 - self.stage_2_window
def run(self, output_dir, dir_is_organized=True):
processed_data_folder = processed_data_dir( self.timeID )
if dir_is_organized:
output_dir = processed_data_folder +'/' + output_dir
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + self.pol_flips_fname )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + self.bad_antennas_fname )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + self.additional_antenna_delays_fname )
station_timing_offsets = read_station_delays( processed_data_folder+'/'+ self.station_delays_fname )
raw_fpaths = filePaths_by_stationName( self.timeID )
self.station_data_files = []
ref_station_i = None
referance_station_set = None
station_i = 0
self.posix_timestamp = None
for station, fpaths in raw_fpaths.items():
if (station in station_timing_offsets) and (station not in self.stations_to_exclude ):
print('opening', station)
raw_data_file = MultiFile_Dal1(fpaths, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays, pol_flips_are_bad=self.pol_flips_are_bad)
self.station_data_files.append( raw_data_file )
raw_data_file.set_station_delay( station_timing_offsets[station] )
raw_data_file.find_and_set_polarization_delay()
if self.posix_timestamp is None:
self.posix_timestamp = raw_data_file.get_timestamp()
elif self.posix_timestamp != raw_data_file.get_timestamp():
print("ERROR: POSIX timestamps are different")
quit()
if self.referance_station is None:
if (referance_station_set is None) or ( int(station[2:]) < int(referance_station_set[2:]) ): ## if no referance station, use one with smallist number
ref_station_i = station_i
referance_station_set = station
elif self.referance_station == station:
ref_station_i = station_i
referance_station_set = station
station_i += 1
## set reference station first
tmp = self.station_data_files[0]
self.station_data_files[0] = self.station_data_files[ ref_station_i ]
self.station_data_files[ ref_station_i ] = tmp
self.referance_station = referance_station_set
## organize antenana info
self.station_antenna_indeces = [] ## 2D list. First index is station_i, second is a local antenna index, value is station antenna index
self.station_antenna_RMS = [] ## same as above, but value is RMS
self.num_total_antennas = 0
current_ant_i = 0
for station_i,stationFile in enumerate(self.station_data_files):
ant_names = stationFile.get_antenna_names()
num_evenAntennas = int( len(ant_names)/2 )
## get RMS planewave fits
if self.antenna_RMS_info is not None:
antenna_RMS_dict = read_planewave_fits(processed_data_folder+'/'+self.antenna_RMS_info, stationFile.get_station_name() )
else:
antenna_RMS_dict = {}
antenna_RMS_array = np.array([ antenna_RMS_dict[ant_name] if ant_name in antenna_RMS_dict else self.default_expected_RMS for ant_name in ant_names if antName_is_even(ant_name) ])
antenna_indeces = np.arange(num_evenAntennas)*2
## remove bad antennas and sort
ant_is_good = np.isfinite( antenna_RMS_array )
antenna_RMS_array = antenna_RMS_array[ ant_is_good ]
antenna_indeces = antenna_indeces[ ant_is_good ]
sorter = np.argsort( antenna_RMS_array )
antenna_RMS_array = antenna_RMS_array[ sorter ]
antenna_indeces = antenna_indeces[ sorter ]
## select only the best!
antenna_indeces = antenna_indeces[:self.max_antennas_per_station]
antenna_RMS_array = antenna_RMS_array[:self.max_antennas_per_station]
## fill info
self.station_antenna_indeces.append( antenna_indeces )
self.station_antenna_RMS.append( antenna_RMS_array )
self.num_total_antennas += len(antenna_indeces)
current_ant_i += len(antenna_indeces)
### output to header!
if not isdir(output_dir):
mkdir(output_dir)
logging_folder = output_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
header_outfile = h5py.File(output_dir + "/header.h5", "w")
header_outfile.attrs["timeID"] = self.timeID
header_outfile.attrs["initial_datapoint"] = self.initial_datapoint
header_outfile.attrs["max_antennas_per_station"] = self.max_antennas_per_station
header_outfile.attrs["referance_station"] = self.station_data_files[0].get_station_name()
header_outfile.attrs["station_delays_fname"] = self.station_delays_fname
header_outfile.attrs["additional_antenna_delays_fname"] = self.additional_antenna_delays_fname
header_outfile.attrs["bad_antennas_fname"] = self.bad_antennas_fname
header_outfile.attrs["pol_flips_fname"] = self.pol_flips_fname
header_outfile.attrs["blocksize"] = self.blocksize
header_outfile.attrs["remove_saturation"] = self.remove_saturation
header_outfile.attrs["remove_RFI"] = self.remove_RFI
header_outfile.attrs["positive_saturation"] = self.positive_saturation
header_outfile.attrs["negative_saturation"] = self.negative_saturation
header_outfile.attrs["saturation_removal_length"] = self.saturation_removal_length
header_outfile.attrs["saturation_half_hann_length"] = self.saturation_half_hann_length
header_outfile.attrs["hann_window_fraction"] = self.hann_window_fraction
header_outfile.attrs["num_zeros_dataLoss_Threshold"] = self.num_zeros_dataLoss_Threshold
header_outfile.attrs["min_amplitude"] = self.min_amplitude
header_outfile.attrs["upsample_factor"] = self.upsample_factor
header_outfile.attrs["max_events_perBlock"] = self.max_events_perBlock
header_outfile.attrs["min_pulse_length_samples"] = self.min_pulse_length_samples
header_outfile.attrs["erasure_length"] = self.erasure_length
header_outfile.attrs["guess_distance"] = self.guess_distance
header_outfile.attrs["kalman_devations_toSearch"] = self.kalman_devations_toSearch
header_outfile.attrs["pol_flips_are_bad"] = self.pol_flips_are_bad
header_outfile.attrs["antenna_RMS_info"] = self.antenna_RMS_info
header_outfile.attrs["default_expected_RMS"] = self.default_expected_RMS
header_outfile.attrs["max_planewave_RMS"] = self.max_planewave_RMS
header_outfile.attrs["stop_chi_squared"] = self.stop_chi_squared
header_outfile.attrs["max_minimize_itters"] = self.max_minimize_itters
header_outfile.attrs["minimize_ftol"] = self.minimize_ftol
header_outfile.attrs["minimize_xtol"] = self.minimize_xtol
header_outfile.attrs["minimize_gtol"] = self.minimize_gtol
header_outfile.attrs["refStat_delay"] = station_timing_offsets[ self.station_data_files[0].get_station_name() ]
header_outfile.attrs["refStat_timestamp"] = self.posix_timestamp#self.station_data_files[0].get_timestamp()
header_outfile.attrs["refStat_sampleNumber"] = self.station_data_files[0].get_nominal_sample_number()
header_outfile.attrs["stations_to_exclude"] = np.array(self.stations_to_exclude, dtype='S')
header_outfile.attrs["polarization_flips"] = np.array(polarization_flips, dtype='S')
header_outfile.attrs["num_stations"] = len(self.station_data_files)
header_outfile.attrs["num_antennas"] = self.num_total_antennas
for stat_i, (stat_file, antenna_indeces, antenna_RMS) in enumerate(zip(self.station_data_files, self.station_antenna_indeces, self.station_antenna_RMS)):
station_group = header_outfile.create_group( str(stat_i) )
station_group.attrs["sname"] = stat_file.get_station_name()
station_group.attrs["num_antennas"] = len( antenna_indeces )
locations = stat_file.get_LOFAR_centered_positions()
total_delays = stat_file.get_total_delays()
ant_names = stat_file.get_antenna_names()
for ant_i, (stat_index, RMS) in enumerate(zip(antenna_indeces, antenna_RMS)):
antenna_group = station_group.create_group( str(ant_i) )
antenna_group.attrs['antenna_name'] = ant_names[stat_index]
antenna_group.attrs['location'] = locations[stat_index]
antenna_group.attrs['delay'] = total_delays[stat_index]
antenna_group.attrs['planewave_RMS'] = RMS
from os import mkdir
from os.path import isdir
## these lines are anachronistic and should be fixed at some point
from LoLIM import utilities
utilities.default_raw_data_loc = "/exp_app2/appexp1/public/raw_data"
utilities.default_processed_data_loc = "/home/brian/processed_files"
if __name__ == "__main__":
timeID = "D20180921T194259.023Z"
output_folder = "/max_over_blocks"
block_size = 2**16
processed_data_dir = processed_data_dir(timeID)
output_fpath = processed_data_dir + output_folder
if not isdir(output_fpath):
mkdir(output_fpath)
#### get paths to raw data by station ####
raw_fpaths = filePaths_by_stationName(timeID)
for station in raw_fpaths.keys():
print("station:", station)
#### open the data for this station ####
TBB_data = MultiFile_Dal1(raw_fpaths[station])
num_antennas = len(TBB_data.get_antenna_names())
#!/usr/bin/env python3 import tkinter as tk import pickle from LoLIM import utilities from LoLIM.utilities import processed_data_dir timeID = "D20190424T210306.154Z" utilities.default_processed_data_loc = "/home/student4/Marten/processed_files" processed_data_folder = processed_data_dir(timeID) ui = tk.Tk() ui.withdraw() clip = bytes.fromhex(ui.clipboard_get()) ui.destroy() data = pickle.loads(clip) with open(processed_data_folder + "/polarization_data/pulseIDs.pkl", 'wb') as f: pickle.dump(data,f,protocol=pickle.HIGHEST_PROTOCOL)
def open_files(self, station, log_func=do_nothing, force_metadata_ant_pos=True):
processed_data_loc = processed_data_dir(self.timeID)
#### open callibration data files ####
### TODO: fix these so they are accounted for in a more standard maner
self.additional_ant_delays = read_antenna_delays( processed_data_loc+'/'+self.additional_antenna_delays_fname )
self.bad_antennas = read_bad_antennas( processed_data_loc+'/'+self.bad_antennas_fname )
self.pol_flips = read_antenna_pol_flips( processed_data_loc+'/'+self.pol_flips_fname )
#### open data files, and find RFI ####
self.station = station
raw_fpaths = filePaths_by_stationName(self.timeID)[station]
self.input_file = MultiFile_Dal1( raw_fpaths, force_metadata_ant_pos=force_metadata_ant_pos )
self.input_file.set_polarization_flips( self.pol_flips )
if self.do_RFI_filtering and self.use_saved_RFI_info:
self.RFI_filters.append( window_and_filter(timeID=self.timeID, sname=station) )
elif self.do_RFI_filtering:
RFI_result = FindRFI(self.input_file, self.block_size, self.initial_RFI_block, self.RFI_num_blocks, self.RFI_max_blocks, verbose=False, figure_location=None)
self.RFI_filters.append( window_and_filter(find_RFI=RFI_result) )
else: ## only basic filtering
self.RFI_filters.append( window_and_filter(blocksize=self.block_size) )
#### find antenna pairs ####
self.antennas_to_use = pairData_evenAnts(self.input_file, self.bad_antennas, self.max_num_antennas )
self.num_antennas = len(self.antennas_to_use)
#### get antenna locations and delays ####
self.antenna_locations = np.zeros((self.num_antennas, 3), dtype=np.double)
self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double)
for ant_i, station_ant_i in enumerate(self.antennas_to_use):
ant_name = self.input_file.get_antenna_names()[ station_ant_i ]
self.antenna_locations[ant_i] = self.input_file.get_LOFAR_centered_positions()[ station_ant_i ]
self.antenna_delays[ant_i] = self.input_file.get_timing_callibration_delays()[ station_ant_i ]
## add additional timing delays
##note this only works for even antennas!
if ant_name in self.additional_ant_delays:
self.antenna_delays[ant_i] += self.additional_ant_delays[ ant_name ][0]
##### find window offsets and lengths ####
self.half_antenna_data_length = np.zeros(self.num_antennas, dtype=np.long)
for ant_i, station_ant_i in enumerate(self.antennas_to_use):
### find max duration to any of the prefered antennas
max_distance = 0.0
for ant_j, station_ant_j in enumerate(self.antennas_to_use):
if ant_j == ant_i:
continue
distance = np.linalg.norm( self.antenna_locations[ ant_j ]-self.antenna_locations[ant_i] )
if distance > max_distance:
max_distance = distance
self.half_antenna_data_length[ant_i] = int(self.pulse_length/2) + int(max_distance/(v_air*5.0E-9))
self.block_exclusion_length = int(0.1*self.block_size) + np.max( self.half_antenna_data_length ) + 1
self.active_block_length = self.block_size - 2*self.block_exclusion_length##the length of block used for looking at data
self.trace_length = 2*np.max(self.half_antenna_data_length )
self.trace_length = 2**( int(np.log2( self.trace_length )) + 1 )
#### allocate some memory ####
self.data_block = np.empty((self.num_antennas,self.block_size), dtype=np.complex)
self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double)
self.stage_1_imager = II_tools.image_data(self.antenna_locations, self.antenna_delays, self.trace_length, self.upsample_factor)
def run_multiple_blocks(self,
output_folder,
initial_datapoint,
start_block,
blocks_per_run,
run_number,
skip_blocks_done=True,
print_to_screen=True):
processed_data_folder = processed_data_dir(self.timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
file_number = 0
while True:
fname = logging_folder + "/log_run_" + str(file_number) + ".txt"
if isfile(fname):
file_number += 1
else:
break
logger_function = logger()
logger_function.set(fname, print_to_screen)
logger_function.take_stdout()
logger_function.take_stderr()
#### TODO!#### improve log all options
logger_function("timeID:", self.timeID)
logger_function("date and time run:", time.strftime("%c"))
logger_function("output folder:", output_folder)
logger_function("block size:", self.block_size)
logger_function("initial data point:", initial_datapoint)
logger_function("first block:", start_block)
logger_function("blocks per run:", blocks_per_run)
logger_function("run number:", run_number)
logger_function("station delay file:", self.station_delays_fname)
logger_function("additional antenna delays file:",
self.additional_antenna_delays_fname)
logger_function("bad antennas file:", self.bad_antennas_fname)
logger_function("pol flips file:", self.pol_flips_fname)
logger_function("bounding box:", self.bounding_box)
logger_function("pulse length:", self.pulse_length)
logger_function("num antennas per station:",
self.num_antennas_per_station)
logger_function("stations excluded:", self.stations_to_exclude)
logger_function("prefered station:", self.prefered_station)
#### open files, save header if necisary ####
self.open_files(logger_function, True)
if run_number == 0:
header_outfile = h5py.File(data_dir + "/header.h5", "w")
self.save_header(header_outfile)
header_outfile.close()
#### run the algorithm!! ####
for block_index in range(run_number * blocks_per_run + start_block,
(run_number + 1) * blocks_per_run +
start_block):
out_fname = data_dir + "/block_" + str(block_index) + ".h5"
tmp_fname = data_dir + "/tmp_" + str(block_index) + ".h5"
if skip_blocks_done and isfile(out_fname):
logger_function("block:", block_index,
"already completed. Skipping")
continue
start_time = time.time()
block_outfile = h5py.File(tmp_fname, "w")
block_start = initial_datapoint + self.active_block_length * block_index
logger_function("starting processing block:", block_index, "at",
block_start)
self.process_block(block_start,
block_index,
block_outfile,
log_func=logger_function)
block_outfile.close()
rename(tmp_fname, out_fname)
logger_function("block done. took:", (time.time() - start_time),
's')
logger_function()
logger_function("done processing")
def plot_blocks(timeID, block_size, block_starts, guess_delays, guess_location = None, bad_stations=[], polarization_flips="polarization_flips.txt",
bad_antennas = "bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", do_remove_saturation = True, do_remove_RFI = True,
positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 5,
referance_station = "CS002", amplify = {}, omitPOL=None):
"""plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays,
if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero.
A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations"""
'''Modified version of plot_multiple_data_all_stations to support usage of clickable GUI for aligning station timings'''
##2
global first_few_names
global defualt_color
global ax
text = []
##~2
if referance_station in guess_delays:
ref_delay = guess_delays[referance_station]
guess_delays = {sname:delay-ref_delay for sname,delay in guess_delays.items()}
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in raw_fpaths.keys() if sname not in bad_stations}
if guess_location is not None:
guess_location = np.array(guess_location)
ref_stat_file = raw_data_files[ referance_station ]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in guess_delays:
guess_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
guess_delays[sname] += (np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_delay)
guess_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9
RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in raw_fpaths.keys() if sname not in bad_stations}
#RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname, blocksize=block_size) for sname in raw_fpaths.keys() if sname not in bad_stations}
data = np.empty(block_size, dtype=np.double)
##3
#fig = plt.figure(figsize=(10, 12))
fig, ax = plt.subplots()
for sname, data_file in raw_data_files.items():
#print("Loading: "+ sname)
if referance_station in sname:
ref_plot = (sname, data_file)
#ref_plot_offset = guess_delays[sname] #currently unsed
else:
plots.append((sname, data_file))
first_few = [ref_plot]
for p in range(0,Nplots-1):
first_few.append(plots.pop(0))
for sname, data_file in first_few:
if sname != referance_station:
first_few_names.append(sname)
##~3
##4
def more_plots(some_data,First,ax,text):
global l,m
ax.clear()
height = 0
t0 = np.arange(block_size)*5.0E-9
transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData)
sname_X_loc = 0.0
n = 0
for sname, data_file in some_data:
print("Plotting: "+sname)
station_delay = -data_file.get_nominal_sample_number()*5.0E-9
if sname in guess_delays:
station_delay = guess_delays[sname]
station_delay_points = int(station_delay/5.0E-9)
RFI_filter = RFI_filters[sname]
ant_names = data_file.get_antenna_names()
num_antenna_pairs = int( len( ant_names )/2 )
peak_height = 0.0
print("Block starts at: "+str(block_starts[0]))
for point in block_starts:
T = t0 + point*5.0E-9
for pair in range(num_antenna_pairs):
data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2)
if do_remove_saturation:
remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter( data )
else:
filtered_data = hilbert(data)
even_HE = np.abs(filtered_data)
data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2+1)
if do_remove_saturation:
remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
if do_remove_RFI:
filtered_data = RFI_filter.filter( data )
else:
filtered_data = hilbert(data)
odd_HE = np.abs(filtered_data)
#ax.plot(T, even_HE + height, 'r')
#ax.plot(T, odd_HE + height, 'g' )
#for cases where data are hard to align due to signal being tiny:
found = False
if len(amplify) > 0:
for key in amplify:
if sname == key:
if omitPOL != 0:
ax.plot(T, amplify[key]*even_HE + height, 'r')
if omitPOL != 1:
ax.plot(T, amplify[key]*odd_HE + height, 'g' )
#ax.plot(T, amplify[key]*even_HE + height, 'r') twinx with block_starts && point as x rather than t! (can I update to latests plot_multiple?)
#ax.plot(T, amplify[key]*odd_HE + height, 'g' )
found = True
if not found:
if omitPOL != 0:
ax.plot(T, even_HE + height, 'r')
if omitPOL != 1:
ax.plot(T, odd_HE + height, 'g' )
#ax2 = ax.twiny()
#xmin = min(block_starts)
#xmax = max(block_starts)
#ax2.set_xlim(xmin, xmax) #can also use ax2.set_xticks(x)
#ax2.xaxis.set_ticks_position('both')
max_even = np.max(even_HE)
if max_even > peak_height:
peak_height = max_even
max_odd = np.max(odd_HE)
if max_odd > peak_height:
peak_height = max_odd
# plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
plt.sca(ax) #somehow this makes it so that the annotations work on all plots except for the first time "Next stations" is clicked
plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform)
ax.ticklabel_format(useOffset=False)
plt.setp(ax.get_xticklabels(), rotation=60, horizontalalignment='right')
plt.subplots_adjust(bottom=0.1,top=0.98,left=0.21)
if ref_plot_peak_location != 0.0:
print("Setting ref peak to " +str(ref_plot_peak_location))
Cpeak = ref_plot_peak_location #peak center, for plotting
Dpeak = 0.0001 # +/- added to peak center for plotting
plt.xlim(Cpeak-Dpeak,Cpeak+Dpeak)
plt.grid(b=True)
height += 2*peak_height
n+=1
##~4
##5
#######################CallBacks############################
#######################CallBacks############################
#######################CallBacks############################
def onclick(event):
global stations_active
global station_buttons
global ref_plot_peak_location
global set_ref
global set_stations
global default_color
ix = event.xdata
iy = event.ydata
#print("click active stations: ", stations_active)
falsify = []
if set_ref:
ref_plot_peak_location = ix
print("Set ref peak to: "+str(ref_plot_peak_location))
set_ref = False
return #return, don't set anything else!
n=0
for s in stations_active.keys():
if stations_active[s] and s !="":
falsify.append(s)
if s not in guess_delays.keys():
guess_delays[s] = 0.0
print ("Set " + s + " station delay to: ", guess_delays[s] + (ix-ref_plot_peak_location))
tmp_guess_delays[s] = guess_delays[s] + (ix - ref_plot_peak_location)
station_buttons[n].color = default_color
station_buttons[n].hovercolor = default_color
n+=1
if len(falsify)>0:
for f in falsify:
stations_active[f] = False
falsify = []
plt.draw()
def nextCB(event):
global ax
global plots
global station_buttons
global stations_active
global StationCallBacks
print("next!")
stations_active = {}
next_few = [ref_plot]
if len(plots) >= Nplots:
for i in range(0,Nplots-1):
next_few.append(plots.pop(0))
elif len(plots) > 0:
bk = len(plots)
for i in range(0,len(plots)):
next_few.append(plots.pop(0))
for i in range(bk,4):
StationCallBacks[i].reset(i,"")
station_buttons[i].label.set_text("")
else:
next_few = []
for i in range(0,Nplots-1):
StationCallBacks[i].reset(i,"")
station_buttons[i].label.set_text("")
print("All stations timings have been set")
ax.clear()
ax.annotate("All stations timings have been set", (0.5, 0.5))
plt.draw()
return
height = 0.0
n = 0
for sname, data_file in next_few:
if sname != referance_station:
first_few_names.append(sname)
StationCallBacks[n].reset(n,sname)
station_buttons[n].label.set_text("Set "+sname)
stations_active[sname] = False
n+=1
more_plots(next_few,False,ax,text)
plt.draw()
def refpCB(event):
global ref_plot_peak_location
global set_ref
print ("Setting ref peak")# to: ",ix)
set_ref = True
def cCB(text):
Cpeak = float(text) #peak center, for plotting
Dpeak = 0.0001 # +/- added to peak center for plotting
plt.xlim(Cpeak+Dpeak,Cpeak-Dpeak)
plt.draw()
def writeCB(event):
print('Writing guess station delays')
file = open('guess_delays.py','w')
file.write("\n self.guess_timing = " +str(ref_plot_peak_location) + "\n")
#file.write(" self.event_index = ____\n\n")
file.write(' self.guess_station_delays = {\n')
for g in guess_delays:
if g in tmp_guess_delays:
'''If the delay has been updated, write the update:'''
file.write('"'+g+'": '+str(tmp_guess_delays[g])+',\n')
else:
"""Else, just write the old delay:"""
file.write('"'+g+'": '+str(guess_delays[g])+',\n')
file.write('}\n')
print('Done!\n')
def press(event):
val = event.key
print('press', val)
'''if val == ('0' or '1' or '2' or '3' or '4'):
val = int(val)
station_buttons[val].color = 'blue'
station_buttons[val] .hovercolor = 'blue'
'''
class StationCB:
global stations_active
def __init__(self,np,station):
self.N = np
self.station = station
def reset(self,np,station):
self.N = np
self.station = station
def stationCB(self,event):
print("Station active: "+self.station)
#print(self.station + ' You pressed: ',event.name)
#---highlight button with color when pressed state, show buttons in correct order
station_buttons[self.N].color = 'blue'
station_buttons[self.N].hovercolor = 'blue'
stations_active[self.station] = True
######################~CallBacks############################
######################~CallBacks############################
######################~CallBacks############################
more_plots(first_few,True,ax,text)
axnext = plt.axes([ 0.05, 0.80, 0.1, 0.07])
bnext = Button(axnext, 'Next\nStations')
bnext.on_clicked(nextCB)
axfile = plt.axes([0.05,0.7, 0.1, 0.07])
bfile = Button(axfile, 'Write\nGuesses')
bfile.on_clicked(writeCB)
axref = plt.axes([0.05, 0.1, 0.13, 0.07])
bref= Button(axref, 'Set\nRef Peak')
bref.on_clicked(refpCB)
#Cbox = plt.axes([0.05, 0.6, 0.1, 0.07])
#text_box = TextBox(Cbox, 'Peak Center', initial="")
#text_box.on_submit(cCB)
start = 0.22
for f in range(0,Nplots-1):
fax = plt.axes([ 0.05, start+f*0.1, 0.13, 0.05])
fbtn = Button(fax, "Set "+first_few_names[f])
stations_active[first_few_names[f]] = False
s = StationCB(f,first_few_names[f])
StationCallBacks.append(s)
station_axes.append(fax)
station_buttons.append(fbtn)
fbtn.on_clicked(s.stationCB)
fig.canvas.mpl_connect('button_press_event', onclick)
fig.canvas.mpl_connect('key_press_event', press)
plt.show()
def __init__(self,
timeID,
guess_delays,
block_size,
initial_block,
num_blocks,
working_folder,
other_folder=None,
max_num_stations=np.inf,
guess_location=None,
bad_stations=[],
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays=None,
do_remove_saturation=True,
do_remove_RFI=True,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
referance_station="CS002",
pulse_length=50,
upsample_factor=4,
min_antenna_amplitude=5,
fix_polarization_delay=True):
#### disable matplotlib shortcuts
plt.rcParams['keymap.save'] = ''
plt.rcParams['keymap.fullscreen'] = ''
plt.rcParams['keymap.home'] = ''
plt.rcParams['keymap.back'] = ''
plt.rcParams['keymap.forward'] = ''
plt.rcParams['keymap.pan'] = ''
plt.rcParams['keymap.zoom'] = ''
plt.rcParams['keymap.save'] = ''
plt.rcParams['keymap.quit_all'] = ''
plt.rcParams['keymap.grid'] = ''
plt.rcParams['keymap.grid_minor'] = ''
plt.rcParams['keymap.yscale'] = ''
plt.rcParams['keymap.xscale'] = ''
plt.rcParams['keymap.all_axes'] = ''
plt.rcParams['keymap.copy'] = ''
#keymap.help : f1 ## display help about active tools
#keymap.quit : ctrl+w, cmd+w, q ## close the current figure
self.pressed_data = None
#guess_location = np.array(guess_location[:3])
self.block_size = block_size
self.num_blocks = num_blocks
self.positive_saturation = positive_saturation
self.negative_saturation = negative_saturation
self.saturation_post_removal_length = saturation_post_removal_length
self.saturation_half_hann_length = saturation_half_hann_length
self.do_remove_saturation = do_remove_saturation
self.do_remove_RFI = do_remove_RFI
self.referance_station = referance_station
#### open data ####
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder +
'/' + polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
if additional_antenna_delays is not None:
additional_antenna_delays = read_antenna_delays(
processed_data_folder + '/' + additional_antenna_delays)
print(
"WARNING: Additional antenna delays should probably be None!")
raw_fpaths = filePaths_by_stationName(timeID)
station_names = [
sname for sname in raw_fpaths.keys() if sname not in bad_stations
]
self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_names}
if fix_polarization_delay:
for sname, file in self.TBB_files.items():
# delays = file.get_timing_callibration_delays()
# print()
# print()
# print(sname)
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
file.find_and_set_polarization_delay()
# delays = file.get_timing_callibration_delays()
# print("after")
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
self.RFI_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in station_names
}
self.edge_width = int(self.block_size * 0.1)
self.display_block_size = self.block_size - 2 * self.edge_width
#### some data things ####
self.antenna_time_corrections = {
sname: TBB_file.get_total_delays()
for sname, TBB_file in self.TBB_files.items()
}
self.ave_ref_ant_delay = np.average(
self.antenna_time_corrections[referance_station])
# ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0]
# for station_corrections in self.antenna_time_corrections.values():
# station_corrections -= ref_antenna_delay
max_num_antennas = np.max(
[len(delays) for delays in self.antenna_time_corrections.values()])
self.temp_data_block = np.empty(
(max_num_antennas, num_blocks, block_size), dtype=np.double)
self.time_array = np.arange(block_size) * 5.0E-9
self.figure = plt.gcf()
self.axes = plt.gca()
#### make managers
guess_delays = {
sname: delay
for sname, delay in guess_delays.items()
if sname not in bad_stations
}
self.clock_offset_manager = clock_offsets(referance_station,
guess_delays, guess_location,
self.TBB_files)
self.frame_manager = frames(initial_block, int(num_blocks / 2),
station_names, max_num_stations,
referance_station, self.display_block_size,
num_blocks)
rem_sat_curry = cur(remove_saturation, 5)(
positive_saturation=positive_saturation,
negative_saturation=negative_saturation,
post_removal_length=saturation_post_removal_length,
half_hann_length=saturation_half_hann_length)
self.pulse_manager = pulses(
pulse_length=pulse_length,
block_size=block_size,
out_folder=working_folder,
other_folder=other_folder,
TBB_data_dict=self.TBB_files,
used_delay_dict=self.antenna_time_corrections,
upsample_factor=upsample_factor,
min_ant_amp=min_antenna_amplitude,
referance_station=referance_station,
RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None),
remove_saturation_curry=(rem_sat_curry
if self.do_remove_saturation else None))
self.plot()
self.mode = 0 ##0 is change delay, 1 is set peak, etc...
self.input_mode = 0 ## 0 is normal. press 's' to enter 1, where numbers are used to enter event. press 's' again to go back to 0, and search for event
self.search_string = ''
self.rect_selector = RectangleSelector(plt.gca(),
self.rectangle_callback,
useblit=True,
rectprops=dict(alpha=0.5,
facecolor='red'),
button=1,
state_modifier_keys={
'move': '',
'clear': '',
'square': '',
'center': ''
})
self.mouse_press = self.figure.canvas.mpl_connect(
'button_press_event', self.mouse_press)
self.mouse_release = plt.gcf().canvas.mpl_connect(
'button_release_event', self.mouse_release)
self.key_press = plt.gcf().canvas.mpl_connect('key_press_event',
self.key_press_event)
self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()]
plt.show()
def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", max_num_planewaves=np.inf,
positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 50, verbose=True):
left_pulse_length = int(pulse_length/2)
right_pulse_length = pulse_length-left_pulse_length
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
TBB_data = MultiFile_Dal1( raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays )
ant_names = TBB_data.get_antenna_names()
antenna_locations = TBB_data.get_LOFAR_centered_positions()
antenna_delays = TBB_data.get_timing_callibration_delays()
num_antenna_pairs = int( len( ant_names )/2 )
if num_antenna_pairs < min_num_antennas:
return np.array([]), np.array([]), np.array([])
RFI_filter = window_and_filter(timeID=timeID, sname=station)
block_size = RFI_filter.blocksize
out_RMS = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_zenith = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
out_azimuth = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
N = 0
data = np.empty( (num_antenna_pairs,block_size), dtype=np.double )
for block in range(initial_block, initial_block+number_of_blocks):
if N >= max_num_planewaves:
break
#### open and filter data
for pair_i in range(num_antenna_pairs):
ant_i = pair_i*2 + polarization
data[pair_i,:] = TBB_data.get_data(block*block_size, block_size, antenna_index=ant_i)
remove_saturation(data[pair_i,:], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
np.abs( RFI_filter.filter( data[pair_i,:] ), out=data[pair_i,:])
#### loop over finding planewaves
i = 0
while i < pulses_per_block:
if N >= max_num_planewaves:
break
pulse_location = 0
pulse_amplitude = 0
## find highest peak
for pair_i, HE in enumerate(data):
loc = np.argmax( HE )
amp = HE[ loc ]
if amp > pulse_amplitude:
pulse_amplitude = amp
pulse_location = loc
## check if is strong enough
if pulse_amplitude < min_amplitude:
break
## get
fitter = locatefier()
for pair_i, HE in enumerate(data):
ant_i = pair_i*2 + polarization
signal = np.array( HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] )
if num_double_zeros(signal, threshold=0.1) == 0:
sample_time = 5.0E-9
if upsample_factor > 1:
sample_time /= upsample_factor
signal = resample(signal, len(signal)*upsample_factor )
peak_finder = parabolic_fit( signal )
peak_time = peak_finder.peak_index*sample_time - antenna_delays[ant_i]
fitter.add_antenna(peak_time, antenna_locations[ant_i])
HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] = 0.0
if fitter.num_measurments() < min_num_antennas:
continue
fitter.prep_for_fitting()
X = brute(fitter.RMS, ranges=[[0,np.pi/2],[0,np.pi*2]], Ns=20)
if X[0] >= np.pi/2:
X[0] = (np.pi/2)*0.99
if X[1] >= np.pi*2:
X[1] = (np.pi*2)*0.99
if X[0] < 0:
X[0] = 0.00000001
if X[1] < 0:
X[1] = 0.00000001
ret = least_squares( fitter.time_fits, X, bounds=[[0,0],[np.pi/2,2*np.pi]], ftol=3e-16, xtol=3e-16, gtol=3e-16, x_scale='jac' )
out_RMS[N] = fitter.RMS(ret.x, 3)
out_zenith[N] = ret.x[0]
out_azimuth[N] = ret.x[1]
N += 1
i += 1
return out_RMS[:N], out_zenith[:N], out_azimuth[:N]
def run_fitter(timeID,
output_folder,
pulse_input_folders,
guess_timings,
souces_to_fit,
guess_source_locations,
source_polarizations,
source_stations_to_exclude,
source_antennas_to_exclude,
bad_ants,
ref_station="CS002",
min_ant_amplitude=10,
num_itters=1000,
error_deviation=0.5E-9,
antenna_error=0.5E-9):
##### holdovers. These globals need to be fixed, so not global....
global station_locations, station_to_antenna_index_list, stations_with_fits, station_to_antenna_index_dict
global referance_station, station_order, sorted_antenna_names, min_antenna_amplitude, ant_locs, bad_antennas
global current_delays_guess, processed_data_folder
referance_station = ref_station
min_antenna_amplitude = min_ant_amplitude
bad_antennas = bad_ants
if referance_station in guess_timings:
ref_T = guess_timings[referance_station]
guess_timings = {
station: T - ref_T
for station, T in guess_timings.items()
if station != referance_station
}
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(
logging_folder +
"/log_out.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("timeID:", timeID)
print("date and time run:", time.strftime("%c"))
print("input folders:", pulse_input_folders)
print("source IDs to fit:", souces_to_fit)
print("guess locations:", guess_source_locations)
print("polarization to use:", source_polarizations)
print("source stations to exclude:", source_stations_to_exclude)
print("source antennas to exclude:", source_antennas_to_exclude)
print("bad antennas:", bad_ants)
print("referance station:", ref_station)
print("guess delays:", guess_timings)
print('pulse error:', error_deviation)
print('antenna error:', antenna_error)
print()
#### open data and data processing stuff ####
print("loading data")
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {
sname: MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
for sname, fpaths in raw_fpaths.items()
if sname in chain(guess_timings.keys(), [referance_station])
}
#### sort antennas and stations ####
station_order = list(
guess_timings.keys()) ## note this doesn't include reference station
sorted_antenna_names = []
station_to_antenna_index_dict = {}
ant_loc_dict = {}
for sname in station_order + [referance_station]:
first_index = len(sorted_antenna_names)
stat_data = raw_data_files[sname]
even_ant_names = stat_data.get_antenna_names()[::2]
even_ant_locs = stat_data.get_LOFAR_centered_positions()[::2]
sorted_antenna_names += even_ant_names
for ant_name, ant_loc in zip(even_ant_names, even_ant_locs):
ant_loc_dict[ant_name] = ant_loc
station_to_antenna_index_dict[sname] = (first_index,
len(sorted_antenna_names))
ant_locs = np.zeros((len(sorted_antenna_names), 3))
for i, ant_name in enumerate(sorted_antenna_names):
ant_locs[i] = ant_loc_dict[ant_name]
station_locations = {
sname: ant_locs[station_to_antenna_index_dict[sname][0]]
for sname in station_order + [referance_station]
}
station_to_antenna_index_list = [
station_to_antenna_index_dict[sname]
for sname in station_order + [referance_station]
]
#### sort the delays guess, and account for station locations ####
current_delays_guess = np.array(
[guess_timings[sname] for sname in station_order])
# original_delays = np.array( current_delays_guess )
#### open info from part 1 ####
input_manager = Part1_input_manager(pulse_input_folders)
#### first we fit the known sources ####
current_sources = []
# next_source = 0
for knownID in souces_to_fit:
source_ID, input_name = input_manager.known_source(knownID)
print("prep fitting:", source_ID)
location = guess_source_locations[source_ID]
## make source
source_to_add = source_object(source_ID, input_name, location,
source_stations_to_exclude[source_ID],
source_antennas_to_exclude[source_ID],
num_itters)
current_sources.append(source_to_add)
polarity = source_polarizations[source_ID]
source_to_add.prep_for_fitting(polarity, guess_timings)
fitter = stochastic_fitter_dt(current_sources)
all_delays = np.empty((num_itters, fitter.num_delays), dtype=np.double)
all_RMSs = np.empty(num_itters, dtype=np.double)
for i in range(num_itters):
all_delays[i, :], all_RMSs[i] = fitter.rerun(error_deviation,
antenna_error)
fitter.employ_result(current_sources)
print('run', i, 'RMS:', all_RMSs[i])
print()
print()
print("station timing errors:")
for i, sname in zip(range(fitter.num_delays), station_order):
print(sname, ":", np.std(all_delays[:, i]))
print()
print()
### get average X, Y, Z for each itteraion
ave_X = np.zeros(num_itters)
ave_Y = np.zeros(num_itters)
ave_Z = np.zeros(num_itters)
for source in current_sources:
ave_X += source.solutions[:, 0]
ave_Y += source.solutions[:, 1]
ave_Z += source.solutions[:, 2]
ave_X /= len(current_sources)
ave_Y /= len(current_sources)
ave_Z /= len(current_sources)
print("absolute location errors:")
print("X", np.std(ave_X), "Y", np.std(ave_Y), "Z", np.std(ave_Z))
print()
print()
print("relative location errors")
for source in current_sources:
source.solutions[:, 0] -= ave_X
source.solutions[:, 1] -= ave_Y
source.solutions[:, 2] -= ave_Z
print("source", source.ID)
print(" ", np.std(source.solutions[:, 0]),
np.std(source.solutions[:, 1]), np.std(source.solutions[:, 2]))
print()
print()
print("average RMS", np.average(all_RMSs), "std of RMS", np.std(all_RMSs))
import json
#Own module imports
from stokes_utils import zenith_to_src
from pulse_calibration import calibrate
from stokes import stokes_params, stokes_plotter
from stokesIO import save_polarization_data
#####################
## SET PATHS ##
#####################
timeID = "D20190424T210306.154Z"
utilities.default_raw_data_loc = "/home/student4/Marten/KAP_data_link/lightning_data"
utilities.default_processed_data_loc = "/home/student4/Marten/processed_files"
processed_data_folder = processed_data_dir(timeID)
station_delay_file = "station_delays.txt"
polarization_flips = "polarization_flips.txt"
bad_antennas = "bad_antennas.txt"
additional_antenna_delays = "ant_delays.txt"
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' +
additional_antenna_delays)
station_timing_offsets = read_station_delays(processed_data_folder + '/' +
station_delay_file)
raw_fpaths = filePaths_by_stationName(timeID)
def save_EventByLoc(timeID, XYZT, station_timing_delays, pulse_index, output_folder, pulse_width=50, min_ant_amp=5, upsample_factor=0,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(logging_folder + "/log_out_"+str(pulse_index)+"_.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("saving traces to file", pulse_index)
print("location:", XYZT)
print("upsample_factor:", upsample_factor)
print()
print()
print("station timing offsets")
print(station_timing_delays)
print()
print()
print("pol flips")
print(polarization_flips)
print()
print()
print("bad antennas")
print(bad_antennas)
print()
print()
print("additional antenna delays")
print(additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays,
station_delay=station_timing_delays[sname]) for sname,fpaths in raw_fpaths.items() if sname in station_timing_delays}
data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_timing_delays}
trace_locator = getTrace_fromLoc( raw_data_files, data_filters )
print()
print()
print("data opened")
out_fname = data_dir + "/potSource_"+str(pulse_index)+".h5"
out_file = h5py.File(out_fname, "w")
for sname in station_timing_delays.keys():
TBB_file = raw_data_files[ sname ]
TBB_file.find_and_set_polarization_delay()
antenna_names = TBB_file.get_antenna_names()
h5_statGroup = out_file.create_group( sname )
print()
print(sname)
for even_ant_i in range(0,len(antenna_names),2):
even_ant_name = antenna_names[ even_ant_i ]
odd_ant_name = antenna_names[ even_ant_i+1 ]
starting_index, PolE_offset, throw, PolE_trace = trace_locator.get_trace_fromLoc(XYZT, even_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
throw, PolO_offset, throw, PolO_trace = trace_locator.get_trace_fromIndex(starting_index, odd_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
PolE_HE = np.abs(PolE_trace)
PolO_HE = np.abs(PolO_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(even_ant_name, (4, pulse_width ), dtype=np.double)
h5_Ant_dataset[0] = np.real(PolE_trace)
h5_Ant_dataset[1] = PolE_HE
h5_Ant_dataset[2] = np.real(PolO_trace)
h5_Ant_dataset[3] = PolO_HE
### note that peak time should NOT account for station timing offsets!
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs['PolE_timeOffset'] = -(PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset'] = -(PolO_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolE_timeOffset_CS'] = (PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset_CS'] = (PolO_offset - station_timing_delays[sname])
sample_time = 5.0E-9
if upsample_factor > 1:
PolE_HE = resample(PolE_HE, len(PolE_HE)*upsample_factor )
PolO_HE = resample(PolO_HE, len(PolO_HE)*upsample_factor )
sample_time /= upsample_factor
if np.max(PolE_HE)> min_ant_amp:
PolE_peak_finder = parabolic_fit( PolE_HE )
h5_Ant_dataset.attrs['PolE_peakTime'] = starting_index*5.0E-9 - (PolE_offset-station_timing_delays[sname]) + PolE_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(PolO_HE)> min_ant_amp:
PolO_peak_finder = parabolic_fit( PolO_HE )
h5_Ant_dataset.attrs['PolO_peakTime'] = starting_index*5.0E-9 - (PolO_offset-station_timing_delays[sname]) + PolO_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", even_ant_name, h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'])
out_file.close()
def __init__(self,
timeID,
guess_delays,
block_size,
initial_block,
num_blocks,
working_folder,
other_folder=None,
max_num_stations=np.inf,
guess_location=None,
bad_stations=[],
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt",
do_remove_saturation=True,
do_remove_RFI=True,
positive_saturation=2046,
negative_saturation=-2047,
saturation_post_removal_length=50,
saturation_half_hann_length=50,
referance_station="CS002",
pulse_length=50,
upsample_factor=4,
min_antenna_amplitude=5,
fix_polarization_delay=True):
self.pressed_data = None
#guess_location = np.array(guess_location[:3])
self.block_size = block_size
self.num_blocks = num_blocks
self.positive_saturation = positive_saturation
self.negative_saturation = negative_saturation
self.saturation_post_removal_length = saturation_post_removal_length
self.saturation_half_hann_length = saturation_half_hann_length
self.do_remove_saturation = do_remove_saturation
self.do_remove_RFI = do_remove_RFI
#### open data ####
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips(processed_data_folder +
'/' + polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(
processed_data_folder + '/' + additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
station_names = [
sname for sname in raw_fpaths.keys() if sname not in bad_stations
]
self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_names}
if fix_polarization_delay:
for sname, file in self.TBB_files.items():
# delays = file.get_timing_callibration_delays()
# print()
# print()
# print(sname)
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
file.find_and_set_polarization_delay()
# delays = file.get_timing_callibration_delays()
# print("after")
# for even_ant_i in range(0,len(delays),2):
# print(" ", delays[even_ant_i+1]-delays[even_ant_i])
#self.RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_names}
self.RFI_filters = {
sname: window_and_filter(timeID=timeID,
sname=sname,
blocksize=block_size)
for sname in station_names
}
#### some data things ####
self.antenna_time_corrections = {
sname: TBB_file.get_total_delays()
for sname, TBB_file in self.TBB_files.items()
}
# ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0]
# for station_corrections in self.antenna_time_corrections.values():
# station_corrections -= ref_antenna_delay
max_num_antennas = np.max(
[len(delays) for delays in self.antenna_time_corrections.values()])
self.temp_data_block = np.empty(
(max_num_antennas, num_blocks, block_size), dtype=np.double)
self.time_array = np.arange(block_size) * 5.0E-9
self.figure = plt.gcf()
self.axes = plt.gca()
#### make managers
guess_delays = {
sname: delay
for sname, delay in guess_delays.items()
if sname not in bad_stations
}
self.clock_offset_manager = clock_offsets(referance_station,
guess_delays, guess_location,
self.TBB_files)
self.frame_manager = frames(initial_block, int(num_blocks / 2),
station_names, max_num_stations,
referance_station, block_size, num_blocks)
rem_sat_curry = cur(remove_saturation, 5)(
positive_saturation=positive_saturation,
negative_saturation=negative_saturation,
post_removal_length=saturation_post_removal_length,
half_hann_length=saturation_half_hann_length)
self.pulse_manager = pulses(
pulse_length=pulse_length,
block_size=block_size,
out_folder=working_folder,
other_folder=other_folder,
TBB_data_dict=self.TBB_files,
used_delay_dict=self.antenna_time_corrections,
upsample_factor=upsample_factor,
min_ant_amp=min_antenna_amplitude,
referance_station=referance_station,
RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None),
remove_saturation_curry=(rem_sat_curry
if self.do_remove_saturation else None))
self.plot()
self.mode = 0 ##0 is change delay, 1 is set peak
self.rect_selector = RectangleSelector(plt.gca(),
self.rectangle_callback,
useblit=True,
rectprops=dict(alpha=0.5,
facecolor='red'),
button=1,
state_modifier_keys={
'move': '',
'clear': '',
'square': '',
'center': ''
})
self.mouse_press = self.figure.canvas.mpl_connect(
'button_press_event', self.mouse_press)
self.mouse_release = plt.gcf().canvas.mpl_connect(
'button_release_event', self.mouse_release)
self.key_press = plt.gcf().canvas.mpl_connect('key_press_event',
self.key_press_event)
self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()]
plt.show()
def __init__(self,
blocksize=None,
find_RFI=None,
timeID=None,
sname=None,
lower_filter=30.0E6,
upper_filter=80.0E6,
half_window_percent=0.1,
time_per_sample=5.0E-9,
filter_roll_width=2.5E6):
self.lower_filter = lower_filter
self.upper_filter = upper_filter
if timeID is not None:
if find_RFI is None:
find_RFI = "/findRFI/findRFI_results"
find_RFI = processed_data_dir(timeID) + find_RFI
if isinstance(find_RFI, str): ## load findRFI data from file
with open(find_RFI, 'rb') as fin:
find_RFI = load(fin)[sname]
self.RFI_data = find_RFI
if self.RFI_data is not None:
if blocksize is None:
blocksize = self.RFI_data['blocksize']
elif self.RFI_data['blocksize'] != blocksize:
print("blocksize and findRFI blocksize must match")
quit()
elif blocksize is None:
print("window and filter needs a blocksize")
## TODO: check block sizes are consistant
quit()
self.blocksize = blocksize
self.half_window_percent = half_window_percent
self.half_hann_window = half_hann_window(blocksize,
half_window_percent)
FFT_frequencies = np.fft.fftfreq(blocksize, d=time_per_sample)
self.bandpass_filter = np.zeros(len(FFT_frequencies), dtype=complex)
self.bandpass_filter[np.logical_and(
FFT_frequencies >= lower_filter,
FFT_frequencies <= upper_filter)] = 1.0
width = filter_roll_width / (FFT_frequencies[1] - FFT_frequencies[0])
if width > 1:
gaussian_weights = gaussian(len(FFT_frequencies), width)
self.bandpass_filter = np.convolve(self.bandpass_filter,
gaussian_weights,
mode='same')
self.bandpass_filter /= np.max(
self.bandpass_filter) ##convolution changes the peak value
self.FFT_frequencies = FFT_frequencies
## completly reject low-frequency bits
self.bandpass_filter[0] = 0.0
self.bandpass_filter[1] = 0.0
##reject RFI
if self.RFI_data is not None:
self.bandpass_filter[self.RFI_data["dirty_channels"]] = 0.0
def plot_station(event_ID, sname_to_plot, plot_bad_antennas,
timeID, output_folder, pulse_input_folders, guess_timings, sources_to_fit, guess_source_locations,
source_polarizations, source_stations_to_exclude, source_antennas_to_exclude, bad_ants,
antennas_to_recalibrate={}, min_ant_amplitude=10, ref_station="CS002"):
processed_data_folder = processed_data_dir( timeID )
file_manager = input_manager( processed_data_folder, pulse_input_folders )
throw, input_Fname = file_manager.known_source( event_ID )
data_file = h5py.File(input_Fname, "r")
stat_group = data_file[ sname_to_plot ]
fpaths = filePaths_by_stationName( timeID )[ sname_to_plot ]
station_file = MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
source_XYZT = guess_source_locations[event_ID]
source_polarization = source_polarizations[event_ID]
antennas_to_exclude = bad_ants + source_antennas_to_exclude[event_ID]
ref_station_delay = 0.0
if ref_station in guess_timings:
ref_station_delay = guess_timings[ref_station]
stat_delay = 0.0
if sname_to_plot != ref_station:
stat_delay = guess_timings[sname_to_plot] - ref_station_delay
even_HEs = []
odd_HEs = []
even_sig = []
odd_sig = []
even_good = []
odd_good = []
even_offet = []
odd_offset = []
even_peakT_offset = []
odd_peakT_offset = []
even_ant_names = []
odd_ant_names = []
peak_amp = 0.0
## first we open the data
even_SSqE = 0.0
odd_SSqE = 0.0
even_N = 0
odd_N = 0
for ant_name, ant_loc in zip(station_file.get_antenna_names(),station_file.get_LOFAR_centered_positions()):
if ant_name not in stat_group:
continue
even_ant_name = ant_name
odd_ant_name = even_antName_to_odd( ant_name )
ant_dataset = stat_group[ant_name]
even_trace = np.array( ant_dataset[0] )
even_HE = np.array( ant_dataset[1] )
odd_trace = np.array( ant_dataset[2] )
odd_HE = np.array( ant_dataset[3] )
even_amp = np.max(even_HE)
odd_amp = np.max(odd_HE)
M = max(even_amp, odd_amp)
if M>peak_amp:
peak_amp = M
travel_delay = np.sqrt( (ant_loc[0]-source_XYZT[0])**2 + (ant_loc[1]-source_XYZT[1])**2 + (ant_loc[2]-source_XYZT[2])**2 )/v_air
travel_delay_odd = travel_delay
if source_polarization==3 and len(source_XYZT)==8:
travel_delay_odd = np.sqrt( (ant_loc[0]-source_XYZT[4])**2 + (ant_loc[1]-source_XYZT[5])**2 + (ant_loc[2]-source_XYZT[6])**2 )/v_air
even_time = - travel_delay - stat_delay -source_XYZT[3]
odd_time = - travel_delay_odd - stat_delay
if even_ant_name in antennas_to_recalibrate:
even_time -= antennas_to_recalibrate[even_ant_name]
if odd_ant_name in antennas_to_recalibrate:
odd_time -= antennas_to_recalibrate[odd_ant_name]
if source_polarization==0 or source_polarization==1 or len(source_XYZT)==4:
odd_time -= source_XYZT[3]
elif (source_polarization==2 or source_polarization==3) and len(source_XYZT)==5:
odd_time -= source_XYZT[4]
elif (source_polarization==2 or source_polarization==3) and len(source_XYZT)==8:
odd_time -= source_XYZT[7]
even_peak_time = ant_dataset.attrs["PolE_peakTime"] + even_time
odd_peak_time = ant_dataset.attrs["PolO_peakTime"] + odd_time
even_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolE_timeOffset_CS']
odd_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolO_timeOffset_CS']
even_is_good = even_amp>min_ant_amplitude
if not even_is_good:
print( even_ant_name, ": amp too low" )
else:
even_is_good = not (even_ant_name in antennas_to_exclude)
if not even_is_good:
print( even_ant_name, ": excluded" )
else:
print( even_ant_name, ": good" )
odd_is_good = odd_amp>min_ant_amplitude
if not odd_is_good:
print( odd_ant_name, ": amp too low" )
else:
odd_is_good = not (odd_ant_name in antennas_to_exclude)
if not odd_is_good:
print( odd_ant_name, ": excluded" )
else:
print( odd_ant_name, ": good" )
even_HEs.append( even_HE )
odd_HEs.append( odd_HE )
even_sig.append( even_trace )
odd_sig.append( odd_trace )
even_good.append( even_is_good )
odd_good.append( odd_is_good )
even_offet.append( even_time )
odd_offset.append( odd_time )
even_ant_names.append( even_ant_name )
odd_ant_names.append( odd_ant_name )
even_peakT_offset.append( even_peak_time )
odd_peakT_offset.append( odd_peak_time )
if even_is_good:
even_SSqE += even_peak_time*even_peak_time
even_N += 1
if odd_is_good:
odd_SSqE += odd_peak_time*odd_peak_time
odd_N += 1
if even_N > 0:
print('even RMS:', np.sqrt(even_SSqE/even_N),end=' ')
if odd_N > 0:
print('odd RMS:', np.sqrt(odd_SSqE/odd_N), end='')
print()
current_offset = 0
for pair_i in range(len(even_HEs)):
even_T = np.arange( len(even_HEs[pair_i]) )*5.0E-9 + even_offet[pair_i]
odd_T = np.arange( len(odd_HEs[pair_i]) )*5.0E-9 + odd_offset[pair_i]
if even_good[pair_i] or plot_bad_antennas:
plt.plot(even_T, even_HEs[pair_i]/peak_amp + current_offset, 'g')
plt.plot(even_T, even_sig[pair_i]/peak_amp + current_offset, 'g')
if even_good[pair_i]:
plt.plot( (even_peakT_offset[pair_i],even_peakT_offset[pair_i]), (current_offset,current_offset+1), 'g' )
if odd_good[pair_i] or plot_bad_antennas:
plt.plot(odd_T, odd_HEs[pair_i]/peak_amp + current_offset, 'm')
plt.plot(odd_T, odd_sig[pair_i]/peak_amp + current_offset, 'm')
if odd_good[pair_i]:
plt.plot( (odd_peakT_offset[pair_i],odd_peakT_offset[pair_i]), (current_offset,current_offset+1), 'm' )
even_str = even_ant_names[pair_i] + " "
odd_str = odd_ant_names[pair_i] + " "
if even_good[pair_i]:
even_str += ': good'
else:
even_str += ': bad'
if odd_good[pair_i]:
odd_str += ': good'
else:
odd_str += ': bad'
plt.annotate(odd_str, xy=(odd_T[0], current_offset+0.3), c='m')
plt.annotate(even_str, xy=(even_T[0], current_offset+0.6), c='g')
current_offset += 2
plt.axvline(x=0)
plt.show()
def recalibrate_pulse(timeID, input_fname, output_fname, set_polarization_delay=True, upsample_factor=4, polarization_flips="polarization_flips.txt"):
processed_data_folder = processed_data_dir(timeID)
raw_fpaths = filePaths_by_stationName(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
input_file = h5py.File(processed_data_folder+'/'+input_fname, "r")
try:
output_file = h5py.File(processed_data_folder+'/'+output_fname, "r+")
except:
output_file = h5py.File(processed_data_folder+'/'+output_fname, "w")
sample_time = 5.0e-9
if upsample_factor>1:
sample_time /= upsample_factor
for sname, h5_statGroup in input_file.items():
out_statGroup = output_file.require_group(sname)
print()
print()
print(sname)
datafile = MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips)
if set_polarization_delay:
datafile.find_and_set_polarization_delay()
antenna_calibrations = { antname:calibration for antname,calibration in zip(datafile.get_antenna_names(), datafile.get_total_delays()) }
for antname, in_antData in h5_statGroup.items():
out_statGroup.copy(in_antData, out_statGroup, name= antname)
out_antData = out_statGroup[antname]
old_even = out_antData.attrs['PolE_timeOffset_CS']
old_odd = out_antData.attrs['PolO_timeOffset_CS']
new_even_delay = antenna_calibrations[antname]
new_odd_delay = antenna_calibrations[ even_antName_to_odd(antname) ]
out_antData.attrs['PolE_timeOffset'] = -new_even_delay ## NOTE: due to historical reasons, there is a sign flip here
out_antData.attrs['PolO_timeOffset'] = -new_odd_delay ## NOTE: due to historical reasons, there is a sign flip here
out_antData.attrs['PolE_timeOffset_CS'] = new_even_delay
out_antData.attrs['PolO_timeOffset_CS'] = new_odd_delay
print(antname, old_even-new_even_delay, old_odd-new_odd_delay)
starting_index = out_antData.attrs['starting_index']
if np.isfinite( out_antData.attrs['PolE_peakTime'] ):
polE_HE = out_antData[1]
if upsample_factor>1:
polE_HE = resample(polE_HE, len(polE_HE)*upsample_factor )
PolE_peak_finder = parabolic_fit( polE_HE )
out_antData.attrs['PolE_peakTime'] = starting_index*5.0E-9 - new_even_delay + PolE_peak_finder.peak_index*sample_time
if np.isfinite( out_antData.attrs['PolO_peakTime'] ):
polO_HE = out_antData[3]
if upsample_factor>1:
polO_HE = resample(polO_HE, len(polO_HE)*upsample_factor )
PolO_peak_finder = parabolic_fit( polO_HE )
out_antData.attrs['PolO_peakTime'] = starting_index*5.0E-9 - new_odd_delay + PolO_peak_finder.peak_index*sample_time
def run_fitter(timeID, output_folder, pulse_input_folders, guess_timings, souces_to_fit, guess_source_locations,
source_polarizations, source_stations_to_exclude, source_antennas_to_exclude, bad_ants,
X_deviations, Y_deviations, Z_deviations,
ref_station="CS002", min_ant_amplitude=10, max_stoch_loop_itters = 2000, min_itters_till_convergence = 100,
initial_jitter_width = 100000E-9, final_jitter_width = 1E-9, cooldown_fraction = 10.0, strong_cooldown_fraction = 100.0,
fitter = "dt"):
##### holdovers. These globals need to be fixed, so not global....
global station_locations, station_to_antenna_index_list, stations_with_fits, station_to_antenna_index_dict
global referance_station, station_order, sorted_antenna_names, min_antenna_amplitude, ant_locs, bad_antennas
global max_itters_per_loop, itters_till_convergence, min_jitter_width, max_jitter_width, cooldown, strong_cooldown
global current_delays_guess, processed_data_folder
referance_station = ref_station
min_antenna_amplitude = min_ant_amplitude
bad_antennas = bad_ants
max_itters_per_loop = max_stoch_loop_itters
itters_till_convergence = min_itters_till_convergence
max_jitter_width = initial_jitter_width
min_jitter_width = final_jitter_width
cooldown = cooldown_fraction
strong_cooldown = strong_cooldown_fraction
if referance_station in guess_timings:
ref_T = guess_timings[referance_station]
guess_timings = {station:T-ref_T for station,T in guess_timings.items() if station != referance_station}
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(logging_folder + "/log_out.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("timeID:", timeID)
print("date and time run:", time.strftime("%c") )
print("input folders:", pulse_input_folders)
print("source IDs to fit:", souces_to_fit)
print("guess locations:", guess_source_locations)
print("polarization to use:", source_polarizations)
print("source stations to exclude:", source_stations_to_exclude)
print("source antennas to exclude:", source_antennas_to_exclude)
print("bad antennas:", bad_ants)
print("referance station:", ref_station)
print("fitter type:", fitter)
print("guess delays:", guess_timings)
print()
print()
#### open data and data processing stuff ####
print("loading data")
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True) for sname,fpaths in raw_fpaths.items() if sname in chain(guess_timings.keys(), [referance_station]) }
#### sort antennas and stations ####
station_order = list(guess_timings.keys())## note this doesn't include reference station
sorted_antenna_names = []
station_to_antenna_index_dict = {}
ant_loc_dict = {}
for sname in station_order + [referance_station]:
first_index = len(sorted_antenna_names)
stat_data = raw_data_files[sname]
even_ant_names = stat_data.get_antenna_names()[::2]
even_ant_locs = stat_data.get_LOFAR_centered_positions()[::2]
sorted_antenna_names += even_ant_names
for ant_name, ant_loc in zip(even_ant_names,even_ant_locs):
ant_loc_dict[ant_name] = ant_loc
station_to_antenna_index_dict[sname] = (first_index, len(sorted_antenna_names))
ant_locs = np.zeros( (len(sorted_antenna_names), 3))
for i, ant_name in enumerate(sorted_antenna_names):
ant_locs[i] = ant_loc_dict[ant_name]
station_locations = {sname:ant_locs[station_to_antenna_index_dict[sname][0]] for sname in station_order + [referance_station]}
station_to_antenna_index_list = [station_to_antenna_index_dict[sname] for sname in station_order + [referance_station]]
#### sort the delays guess, and account for station locations ####
current_delays_guess = np.array([guess_timings[sname] for sname in station_order])
original_delays = np.array( current_delays_guess )
#### open info from part 1 ####
input_manager = Part1_input_manager( pulse_input_folders )
#### first we fit the known sources ####
current_sources = []
# next_source = 0
for knownID in souces_to_fit:
source_ID, input_name = input_manager.known_source( knownID )
print("prep fitting:", source_ID)
location = guess_source_locations[source_ID]
## make source
source_to_add = source_object(source_ID, input_name, location, source_stations_to_exclude[source_ID], source_antennas_to_exclude[source_ID] )
current_sources.append( source_to_add )
polarity = source_polarizations[source_ID]
source_to_add.prep_for_fitting(polarity)
print()
print("fitting known sources")
if fitter!='dt':
print("only dt")
quit()
first_source = current_sources[0]
original_XYZT = np.array( first_source.guess_XYZT )
if X_deviations is not None:
RMS_values = np.empty( len(X_deviations) )
first_source.guess_XYZT[:] = original_XYZT
for i in range(len(X_deviations)):
first_source.guess_XYZT[0] = original_XYZT[0] + X_deviations[i]
fitter = stochastic_fitter_dt(current_sources)
RMS_values[i] = fitter.RMS
print('X', X_deviations[i], fitter.RMS)
plt.plot(X_deviations, RMS_values)
plt.show()
if Y_deviations is not None:
RMS_values = np.empty( len(Y_deviations) )
first_source.guess_XYZT[:] = original_XYZT
for i in range(len(Y_deviations)):
first_source.guess_XYZT[1] = original_XYZT[1] + Y_deviations[i]
fitter = stochastic_fitter_dt(current_sources)
RMS_values[i] = fitter.RMS
print('Y', Y_deviations[i], fitter.RMS)
plt.plot(Y_deviations, RMS_values)
plt.show()
if Z_deviations is not None:
RMS_values = np.empty( len(Z_deviations) )
first_source.guess_XYZT[:] = original_XYZT
for i in range(len(Z_deviations)):
first_source.guess_XYZT[2] = original_XYZT[2] + Z_deviations[i]
fitter = stochastic_fitter_dt(current_sources)
RMS_values[i] = fitter.RMS
print('Z', Z_deviations[i], fitter.RMS)
plt.plot(Z_deviations, RMS_values)
plt.show()
def setup(self):
if self.ref_station in self.guess_timings:
ref_T = self.guess_timings[self.ref_station]
self.guess_timings = {
station: T - ref_T
for station, T in self.guess_timings.items()
if station != self.ref_station
}
processed_data_folder = processed_data_dir(self.timeID)
data_dir = processed_data_folder + "/" + self.output_folder
if not isdir(data_dir):
mkdir(data_dir)
#Setup logger
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
self.log = logger()
self.log.set(logging_folder + "/log_out.txt"
) ## TODo: save all output to a specific output folder
self.log.take_stderr()
self.log.take_stdout()
print("timeID:", self.timeID)
print("date and time run:", time.strftime("%c"))
print("input folders:", self.pulse_input_folders)
print("source IDs to fit:", self.sources_to_fit)
print("guess locations:", self.guess_source_locations)
print("polarization to use:", self.source_polarizations)
print("source stations to exclude:", self.source_stations_to_exclude)
print("source antennas to exclude:", self.source_antennas_to_exclude)
print("bad antennas:", self.bad_ants)
print("referance station:", self.ref_station)
print("guess delays:", self.guess_timings)
print()
print()
#### open data and data processing stuff ####
print("loading data")
## needed for ant locs
raw_fpaths = filePaths_by_stationName(self.timeID)
raw_data_files = {
sname: MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
for sname, fpaths in raw_fpaths.items()
if sname in chain(self.guess_timings.keys(), [self.ref_station])
}
#### sort antennas and stations ####
self.station_order = list(
self.guess_timings.keys()) + [self.ref_station]
self.sorted_antenna_names = []
self.station_to_antenna_index_dict = {}
self.ant_loc_dict = {}
for sname in self.station_order:
first_index = len(self.sorted_antenna_names)
stat_data = raw_data_files[sname]
ant_names = stat_data.get_antenna_names()
stat_ant_locs = stat_data.get_LOFAR_centered_positions()
self.sorted_antenna_names += ant_names
for ant_name, ant_loc in zip(ant_names, stat_ant_locs):
self.ant_loc_dict[ant_name] = ant_loc
self.station_to_antenna_index_dict[sname] = (
first_index, len(self.sorted_antenna_names))
self.ant_locs = np.zeros((len(self.sorted_antenna_names), 3))
for i, ant_name in enumerate(self.sorted_antenna_names):
self.ant_locs[i] = self.ant_loc_dict[ant_name]
self.station_locations = {
sname: self.ant_locs[self.station_to_antenna_index_dict[sname][0]]
for sname in self.station_order
}
self.station_to_antenna_index_list = [
self.station_to_antenna_index_dict[sname]
for sname in self.station_order
]
#### sort the delays guess, and account for station locations ####
self.current_delays_guess = np.array(
[self.guess_timings[sname] for sname in self.station_order[:-1]])
self.original_delays = np.array(self.current_delays_guess)
self.ant_recalibrate_order = np.array([
i for i, antname in enumerate(self.sorted_antenna_names)
if antname in self.antennas_to_recalibrate.keys()
],
dtype=np.int)
self.ant_recalibrate_guess = np.array([
self.antennas_to_recalibrate[self.sorted_antenna_names[i]]
for i in self.ant_recalibrate_order
])
self.station_indeces = np.empty(len(self.sorted_antenna_names),
dtype=np.int)
for station_index, index_range in enumerate(
self.station_to_antenna_index_list
): ## note this DOES include ref stat
first, last = index_range
self.station_indeces[first:last] = station_index
#### now we open the pulses ####
input_manager = Part1_input_manager(processed_data_folder,
self.pulse_input_folders)
self.current_sources = []
for knownID in self.sources_to_fit:
source_ID, input_name = input_manager.known_source(knownID)
print("prep fitting:", source_ID)
polarity = self.source_polarizations[source_ID]
source_to_add = source_object(
source_ID, input_name,
self.source_stations_to_exclude[source_ID],
self.source_antennas_to_exclude[source_ID])
source_to_add.prep_for_fitting(polarity, self)
#quit()
self.current_sources.append(source_to_add)
self.num_sources = len(self.current_sources)
self.num_delays = len(self.original_delays)
self.num_RecalAnts = len(self.ant_recalibrate_order)
self.num_antennas = len(self.sorted_antenna_names)
self.num_measurments = self.num_sources * self.num_antennas
self.current_solution = np.zeros(self.num_delays + self.num_RecalAnts +
4 * self.num_sources)
self.current_solution[:self.num_delays] = self.current_delays_guess
self.current_solution[self.num_delays:self.num_delays +
self.num_RecalAnts] = self.ant_recalibrate_guess
self.fitter = delay_fitter(self.ant_locs, self.station_indeces,
self.ant_recalibrate_order,
self.num_sources, self.num_delays)
initial_param_i = self.num_delays + self.num_RecalAnts
self.num_DOF = -self.num_delays - self.num_RecalAnts - 4 * self.num_sources
for i, PSE in enumerate(self.current_sources):
self.current_solution[initial_param_i + 4 * i:initial_param_i + 4 *
(i + 1)] = self.guess_source_locations[
PSE.ID]
self.num_DOF += PSE.num_data()
self.fitter.set_event(PSE.pulse_times)
def __init__(self, input_folder, timeID=None):
if timeID is not None:
processed_data_folder = processed_data_dir( timeID )
input_folder = processed_data_folder + '/' + input_folder
self.input_folder = input_folder
header_infile = h5py.File(input_folder + "/header.h5", "r")
#### input settings ####
self.timeID = header_infile.attrs['timeID']
self.initial_datapoint = header_infile.attrs["initial_datapoint"]
self.max_antennas_per_station = header_infile.attrs["max_antennas_per_station"]
if "referance_station" in header_infile.attrs:
self.referance_station = header_infile.attrs["referance_station"]#.decode()
self.station_delays_fname = header_infile.attrs["station_delays_fname"]#.decode()
self.additional_antenna_delays_fname = header_infile.attrs["additional_antenna_delays_fname"]#.decode()
self.bad_antennas_fname = header_infile.attrs["bad_antennas_fname"]#.decode()
self.pol_flips_fname = header_infile.attrs["pol_flips_fname"]#.decode()
self.blocksize = header_infile.attrs["blocksize"]
self.remove_saturation = header_infile.attrs["remove_saturation"]
self.remove_RFI = header_infile.attrs["remove_RFI"]
self.positive_saturation = header_infile.attrs["positive_saturation"]
self.negative_saturation = header_infile.attrs["negative_saturation"]
self.saturation_removal_length = header_infile.attrs["saturation_removal_length"]
self.saturation_half_hann_length = header_infile.attrs["saturation_half_hann_length"]
self.hann_window_fraction = header_infile.attrs["hann_window_fraction"]
self.num_zeros_dataLoss_Threshold = header_infile.attrs["num_zeros_dataLoss_Threshold"]
self.min_amplitude = header_infile.attrs["min_amplitude"]
self.upsample_factor = header_infile.attrs["upsample_factor"]
self.max_events_perBlock = header_infile.attrs["max_events_perBlock"]
self.min_pulse_length_samples = header_infile.attrs["min_pulse_length_samples"]
self.erasure_length = header_infile.attrs["erasure_length"]
self.guess_distance = header_infile.attrs["guess_distance"]
self.kalman_devations_toSearch = header_infile.attrs["kalman_devations_toSearch"]
self.pol_flips_are_bad = header_infile.attrs["pol_flips_are_bad"]
self.antenna_RMS_info = header_infile.attrs["antenna_RMS_info"]
self.default_expected_RMS = header_infile.attrs["default_expected_RMS"]
self.max_planewave_RMS = header_infile.attrs["max_planewave_RMS"]
self.stop_chi_squared = header_infile.attrs["stop_chi_squared"]
self.max_minimize_itters = header_infile.attrs["max_minimize_itters"]
self.minimize_ftol = header_infile.attrs["minimize_ftol"]
self.minimize_xtol = header_infile.attrs["minimize_xtol"]
self.minimize_gtol = header_infile.attrs["minimize_gtol"]
self.refStat_delay = header_infile.attrs["refStat_delay"]
self.refStat_timestamp = header_infile.attrs["refStat_timestamp"]
self.refStat_sampleNumber = header_infile.attrs["refStat_sampleNumber"]
self.stations_to_exclude = header_infile.attrs["stations_to_exclude"]
self.stations_to_exclude = [sname.decode() for sname in self.stations_to_exclude]
#### processed info ####
self.pol_flips = header_infile.attrs["polarization_flips"]
self.pol_flips = [name.decode() for name in self.pol_flips]
self.num_stations = header_infile.attrs["num_stations"]
self.num_total_antennas = header_infile.attrs["num_antennas"]
self.station_names = [ None ]*self.num_stations
self.antenna_info = [ None ]*self.num_stations ## each item will be a list of antenna info
for stat_i, stat_group in header_infile.items():
stat_i = int(stat_i)
sname = stat_group.attrs["sname"]#.decode()
num_antennas = stat_group.attrs["num_antennas"]
ant_info = [ None ]*num_antennas
for ant_i, ant_group in stat_group.items():
ant_i = int(ant_i)
new_antenna_object = self.antenna_info_object()
new_antenna_object.sname = sname
new_antenna_object.ant_name = ant_group.attrs["antenna_name"]#.decode()
new_antenna_object.delay = ant_group.attrs["delay"]
new_antenna_object.planewave_RMS = ant_group.attrs["planewave_RMS"]
new_antenna_object.location = ant_group.attrs["location"]
ant_info[ant_i] = new_antenna_object
self.station_names[ stat_i ] = sname
self.antenna_info[ stat_i ] = ant_info
def plot_all_stations(event_ID,
timeID, output_folder, pulse_input_folders, guess_timings, sources_to_fit, guess_source_locations,
source_polarizations, source_stations_to_exclude, source_antennas_to_exclude, bad_ants,
antennas_to_recalibrate={}, min_ant_amplitude=10, ref_station="CS002"):
processed_data_folder = processed_data_dir( timeID )
file_manager = input_manager( processed_data_folder, pulse_input_folders )
throw, input_Fname = file_manager.known_source( event_ID )
data_file = h5py.File(input_Fname, "r")
fpaths = filePaths_by_stationName( timeID )
source_XYZT = guess_source_locations[event_ID]
source_polarization = source_polarizations[event_ID]
antennas_to_exclude = bad_ants + source_antennas_to_exclude[event_ID]
ref_station_delay = 0.0
if ref_station in guess_timings:
ref_station_delay = guess_timings[ref_station]
current_offset = 0
for sname in data_file.keys():
if (sname not in guess_timings and sname !=ref_station) or (sname in source_stations_to_exclude[event_ID]):
continue
stat_group = data_file[ sname ]
station_file = MultiFile_Dal1(fpaths[sname], force_metadata_ant_pos=True)
stat_delay = 0.0
if sname != ref_station:
stat_delay = guess_timings[sname] - ref_station_delay
even_HEs = []
odd_HEs = []
even_offet = []
odd_offset = []
even_peakT_offset = []
odd_peakT_offset = []
peak_amp = 0.0
## first we open the data
for ant_name, ant_loc in zip(station_file.get_antenna_names(),station_file.get_LOFAR_centered_positions()):
if ant_name not in stat_group:
continue
even_ant_name = ant_name
odd_ant_name = even_antName_to_odd( ant_name )
ant_dataset = stat_group[ant_name]
# even_trace = np.array( ant_dataset[0] )
even_HE = np.array( ant_dataset[1] )
# odd_trace = np.array( ant_dataset[2] )
odd_HE = np.array( ant_dataset[3] )
even_amp = np.max(even_HE)
odd_amp = np.max(odd_HE)
travel_delay = np.sqrt( (ant_loc[0]-source_XYZT[0])**2 + (ant_loc[1]-source_XYZT[1])**2 + (ant_loc[2]-source_XYZT[2])**2 )/v_air
total_correction = travel_delay + stat_delay
even_time = - total_correction - source_XYZT[3]
odd_time = - total_correction
if even_ant_name in antennas_to_recalibrate:
even_time -= antennas_to_recalibrate[even_ant_name]
if odd_ant_name in antennas_to_recalibrate:
odd_time -= antennas_to_recalibrate[odd_ant_name]
if source_polarization==2 and len(source_XYZT)==5:
odd_time -= source_XYZT[4]
else:
odd_time -= source_XYZT[3]
even_peak_time = ant_dataset.attrs["PolE_peakTime"] + even_time
odd_peak_time = ant_dataset.attrs["PolO_peakTime"] + odd_time
even_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolE_timeOffset_CS']
odd_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolO_timeOffset_CS']
even_is_good = (even_amp>min_ant_amplitude) and not (even_ant_name in antennas_to_exclude)
odd_is_good = (odd_amp>min_ant_amplitude) and not (odd_ant_name in antennas_to_exclude)
if (not even_is_good) and (not odd_is_good):
continue
M = max(even_amp, odd_amp)
if M>peak_amp:
peak_amp = M
if even_is_good and source_polarization!=1:
even_HEs.append( even_HE )
else:
even_HEs.append( None )
if odd_is_good and source_polarization!=0:
odd_HEs.append( odd_HE )
else:
odd_HEs.append( None )
even_offet.append( even_time )
odd_offset.append( odd_time )
even_peakT_offset.append( even_peak_time )
odd_peakT_offset.append( odd_peak_time )
earliest_T = np.inf
for pair_i in range(len(even_HEs)):
if even_HEs[pair_i] is not None:
even_T = np.arange( len(even_HEs[pair_i]) )*5.0E-9 + even_offet[pair_i]
if even_T[0] < earliest_T:
earliest_T = even_T[0]
p = plt.plot(even_T, even_HEs[pair_i]/peak_amp + current_offset)
color = p[0].get_color()
plt.plot( (even_peakT_offset[pair_i],even_peakT_offset[pair_i]), (current_offset,current_offset+1), color=color )
if odd_HEs[pair_i] is not None:
odd_T = np.arange( len(odd_HEs[pair_i]) )*5.0E-9 + odd_offset[pair_i]
if odd_T[0] < earliest_T:
earliest_T = odd_T[0]
p = plt.plot(odd_T, odd_HEs[pair_i]/peak_amp + current_offset)
color = p[0].get_color()
plt.plot( (odd_peakT_offset[pair_i],odd_peakT_offset[pair_i]), (current_offset,current_offset+1), color=color )
plt.annotate(sname, xy=(earliest_T,current_offset+0.5))
current_offset += 1
plt.axvline(x=0)
plt.show()
def save_Pulse(timeID, output_folder, event_index, pulse_time, station_delays, skip_stations=[], window_width=7E-6, pulse_width=50, block_size=2**16,
min_ant_amp=5.0, guess_flash_location=None, referance_station="CS002", polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
"""Desined to be used in conjuction with plot_multiple_data_all_stations.py. Given a pulse_time, and window_width, it saves a pulse on every antenna that is
centered on the largest peak in the window, with a width of pulse_width. station_delays doesn't need to be accurate, just should be the same as used in plot_multiple_data_all_stations.
Same for referance_station, and guess_flash_location. Note that this will save the pulse under an index, and will overwrite any other pulse saved under that index in the output folder"""
if referance_station in station_delays:
ref_delay = station_delays[referance_station]
station_delays = {sname:delay-ref_delay for sname,delay in station_delays.items()}
#### setup directory variables ####
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log.set(logging_folder + "/log_event_"+str(event_index)+".txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("pulse time:", pulse_time)
print("window_width:", window_width)
print("pulse_width:", pulse_width)
print("skip stations:", skip_stations)
print("guess_flash_location:", guess_flash_location)
print("input delays:")
print( station_delays)
print()
##station data
print()
print("opening station data")
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_delays.keys()}
data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_delays.keys()}
trace_locator = getTrace_fromLoc( raw_data_files, data_filters, {sname:0.0 for sname in station_delays.keys()} )
if guess_flash_location is not None:
guess_flash_location = np.array(guess_flash_location)
ref_stat_file = raw_data_files[ referance_station ]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in station_delays:
station_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
station_delays[sname] += (np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_delay)
station_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9
print()
print("used apparent delays:")
print(station_delays)
print()
out_fname = data_dir + "/potSource_"+str(event_index)+".h5"
out_file = h5py.File(out_fname, "w")
half_pulse_width = int(0.5*pulse_width)
window_width_samples = int(window_width/5.0E-9)
for sname in station_delays.keys():
if (not np.isfinite(station_delays[sname])) or sname in skip_stations:
continue
h5_statGroup = out_file.create_group( sname )
antenna_names = raw_data_files[sname].get_antenna_names()
data_arrival_index = int((pulse_time + station_delays[sname] - window_width*0.5)/5.0E-9)
print()
print(sname)
for even_ant_i in range(0, len(antenna_names), 2):
#### get window and find peak ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i], window_width_samples)
throw, odd_total_time_offset, throw, odd_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i+1], window_width_samples)
even_HE = np.abs(even_trace)
even_good = False
if np.max( even_HE ) > min_ant_amp:
even_good = True
even_para_fit = parabolic_fit( even_HE )
odd_HE = np.abs(odd_trace)
odd_good = False
if np.max( odd_HE ) > min_ant_amp:
odd_good = True
odd_para_fit = parabolic_fit( odd_HE )
if (not even_good) and (not odd_good):
continue
elif even_good and (not odd_good):
center_index = int(even_para_fit.peak_index)
elif (not even_good) and odd_good:
center_index = int(odd_para_fit.peak_index)
else: ## both good
even_amp = even_HE[ int(even_para_fit.peak_index) ]
odd_amp = odd_HE[ int(odd_para_fit.peak_index) ]
if even_amp > odd_amp:
center_index = int(even_para_fit.peak_index)
else:
center_index = int(odd_para_fit.peak_index)
#### now get data centered on peak. re-get data, as peak may be near edge ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i], half_pulse_width*2)
throw, odd_total_time_offset , throw, odd_trace = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i+1], half_pulse_width*2)
# even_trace = even_trace[center_index-half_pulse_width:center_index+half_pulse_width]
# odd_trace = odd_trace[center_index-half_pulse_width:center_index+half_pulse_width]
even_HE = np.abs(even_trace)
odd_HE = np.abs(odd_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(antenna_names[even_ant_i], (4, half_pulse_width*2), dtype=np.double)
h5_Ant_dataset[0] = np.real( even_trace )
h5_Ant_dataset[1] = even_HE
h5_Ant_dataset[2] = np.real( odd_trace )
h5_Ant_dataset[3] = odd_HE
WARNING: not sure this is correct. even_total_time_offset includes station delay
starting_index = data_arrival_index+center_index-half_pulse_width
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs['PolE_timeOffset'] = -even_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
h5_Ant_dataset.attrs['PolO_timeOffset'] = -odd_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
if np.max(even_HE) > min_ant_amp:
# even_HE = resample(even_HE, len(even_HE)*8 )
# even_para_fit = parabolic_fit( even_HE )
# h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - even_total_time_offset
even_para_fit = parabolic_fit( even_HE )
h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index + starting_index)*5.0E-9 - even_total_time_offset
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(odd_HE) > min_ant_amp:
# odd_HE = resample(odd_HE, len(odd_HE)*8 )
# odd_para_fit = parabolic_fit( odd_HE )
# h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - odd_total_time_offset
odd_para_fit = parabolic_fit( odd_HE )
h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index + starting_index)*5.0E-9 - odd_total_time_offset
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", antenna_names[even_ant_i], (data_arrival_index+center_index)*5.0E-9 - station_delays[sname], h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'] )
out_file.close()
