def run_process(process_i):
inHeader = read_header(out_folder, timeID)
log_fname = inHeader.next_log_file()
logger_function = logger()
logger_function.set(log_fname, False)
logger_function.take_stdout()
logger_function.take_stderr()
logger_function("process", process_i)
logger_function("date and time run:", time.strftime("%c"))
mapper = iterative_mapper(inHeader, logger_function)
current_datapoint = 0
current_run = 0
while current_datapoint < final_datapoint:
for blockJ in range(num_consecutive_blocks):
block = first_block + blockJ + process_i * num_consecutive_blocks + current_run * num_consecutive_blocks * num_processes
mapper.process_block(block, logger_function)
current_datapoint = inHeader.initial_datapoint + (
block + 1) * mapper.usable_block_length
if current_datapoint > final_datapoint:
break
current_run += 1
logger_function("done!")
def setup(self):
if self.ref_station in self.guess_timings:
ref_T = self.guess_timings[self.ref_station]
self.guess_timings = {
station: T - ref_T
for station, T in self.guess_timings.items()
if station != self.ref_station
}
processed_data_folder = processed_data_dir(self.timeID)
data_dir = processed_data_folder + "/" + self.output_folder
if not isdir(data_dir):
mkdir(data_dir)
#Setup logger
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
self.log = logger()
self.log.set(logging_folder + "/log_out.txt"
) ## TODo: save all output to a specific output folder
self.log.take_stderr()
self.log.take_stdout()
print("timeID:", self.timeID)
print("date and time run:", time.strftime("%c"))
print("input folders:", self.pulse_input_folders)
print("source IDs to fit:", self.sources_to_fit)
print("guess locations:", self.guess_source_locations)
print("polarization to use:", self.source_polarizations)
print("source stations to exclude:", self.source_stations_to_exclude)
print("source antennas to exclude:", self.source_antennas_to_exclude)
print("bad antennas:", self.bad_ants)
print("referance station:", self.ref_station)
print("guess delays:", self.guess_timings)
print()
print()
#### open data and data processing stuff ####
print("loading data")
## needed for ant locs
raw_fpaths = filePaths_by_stationName(self.timeID)
raw_data_files = {
sname: MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
for sname, fpaths in raw_fpaths.items()
if sname in chain(self.guess_timings.keys(), [self.ref_station])
}
#### sort antennas and stations ####
self.station_order = list(
self.guess_timings.keys()) + [self.ref_station]
self.sorted_antenna_names = []
self.station_to_antenna_index_dict = {}
self.ant_loc_dict = {}
for sname in self.station_order:
first_index = len(self.sorted_antenna_names)
stat_data = raw_data_files[sname]
ant_names = stat_data.get_antenna_names()
stat_ant_locs = stat_data.get_LOFAR_centered_positions()
self.sorted_antenna_names += ant_names
for ant_name, ant_loc in zip(ant_names, stat_ant_locs):
self.ant_loc_dict[ant_name] = ant_loc
self.station_to_antenna_index_dict[sname] = (
first_index, len(self.sorted_antenna_names))
self.ant_locs = np.zeros((len(self.sorted_antenna_names), 3))
for i, ant_name in enumerate(self.sorted_antenna_names):
self.ant_locs[i] = self.ant_loc_dict[ant_name]
self.station_locations = {
sname: self.ant_locs[self.station_to_antenna_index_dict[sname][0]]
for sname in self.station_order
}
self.station_to_antenna_index_list = [
self.station_to_antenna_index_dict[sname]
for sname in self.station_order
]
#### sort the delays guess, and account for station locations ####
self.current_delays_guess = np.array(
[self.guess_timings[sname] for sname in self.station_order[:-1]])
self.original_delays = np.array(self.current_delays_guess)
self.ant_recalibrate_order = np.array([
i for i, antname in enumerate(self.sorted_antenna_names)
if antname in self.antennas_to_recalibrate.keys()
],
dtype=np.int)
self.ant_recalibrate_guess = np.array([
self.antennas_to_recalibrate[self.sorted_antenna_names[i]]
for i in self.ant_recalibrate_order
])
self.station_indeces = np.empty(len(self.sorted_antenna_names),
dtype=np.int)
for station_index, index_range in enumerate(
self.station_to_antenna_index_list
): ## note this DOES include ref stat
first, last = index_range
self.station_indeces[first:last] = station_index
#### now we open the pulses ####
input_manager = Part1_input_manager(processed_data_folder,
self.pulse_input_folders)
self.current_sources = []
for knownID in self.sources_to_fit:
source_ID, input_name = input_manager.known_source(knownID)
print("prep fitting:", source_ID)
polarity = self.source_polarizations[source_ID]
source_to_add = source_object(
source_ID, input_name,
self.source_stations_to_exclude[source_ID],
self.source_antennas_to_exclude[source_ID])
source_to_add.prep_for_fitting(polarity, self)
#quit()
self.current_sources.append(source_to_add)
self.num_sources = len(self.current_sources)
self.num_delays = len(self.original_delays)
self.num_RecalAnts = len(self.ant_recalibrate_order)
self.num_antennas = len(self.sorted_antenna_names)
self.num_measurments = self.num_sources * self.num_antennas
self.current_solution = np.zeros(self.num_delays + self.num_RecalAnts +
4 * self.num_sources)
self.current_solution[:self.num_delays] = self.current_delays_guess
self.current_solution[self.num_delays:self.num_delays +
self.num_RecalAnts] = self.ant_recalibrate_guess
self.fitter = delay_fitter(self.ant_locs, self.station_indeces,
self.ant_recalibrate_order,
self.num_sources, self.num_delays)
initial_param_i = self.num_delays + self.num_RecalAnts
self.num_DOF = -self.num_delays - self.num_RecalAnts - 4 * self.num_sources
for i, PSE in enumerate(self.current_sources):
self.current_solution[initial_param_i + 4 * i:initial_param_i + 4 *
(i + 1)] = self.guess_source_locations[
PSE.ID]
self.num_DOF += PSE.num_data()
self.fitter.set_event(PSE.pulse_times)
processed_data_folder = processed_data_dir(timeID)
output_fpath = processed_data_folder + output_log_folder
if not isdir(output_fpath):
mkdir(output_fpath)
if not isdir(history_folder):
mkdir(history_folder)
fpaths = filePaths_by_stationName(timeID)
polarization_flips = processed_data_folder + '/' + "polarization_flips.txt"
bad_antennas = processed_data_folder + '/' + "bad_antennas.txt"
# additional_antenna_delays = processed_data_folder + '/' + "ant_delays.txt"
additional_antenna_delays = None
station_log = logger()
station_log.take_stdout()
#### open the station
print("processing", station)
station_log.set(output_fpath + '/' + station + '_log.txt')
TBB_data = MultiFile_Dal1(fpaths[station],
polarization_flips=polarization_flips,
bad_antennas=bad_antennas,
additional_ant_delays=additional_antenna_delays)
if TBB_data.needs_metadata():
print(station, "does not have delay metadata. This will not work.")
## these lines are anachronistic and should be fixed at some point
from LoLIM import utilities
utilities.default_raw_data_loc = "/exp_app2/appexp1/lightning_data"
utilities.default_processed_data_loc = "/home/brian/processed_files"
out_folder = 'iterMapper_50_CS003'
timeID = 'D20170929T202255.000Z'
final_datapoint = 5500 * (2**16
) ## this may be exceeded by upto one block size
inHeader = read_header(out_folder, timeID)
log_fname = inHeader.next_log_file()
logger_function = logger()
logger_function.set(log_fname, True)
logger_function.take_stdout()
logger_function.take_stderr()
logger_function("date and time run:", time.strftime("%c"))
mapper = iterative_mapper(inHeader, logger_function)
current_datapoint = 0
current_block = 0
while current_datapoint < final_datapoint:
mapper.process_block(current_block, logger_function)
current_datapoint = inHeader.initial_datapoint + (
output_folder = "/find_percent_data_saturated"
initial_block = 3400
final_block = 4400
positive_saturation = 2046
negative_saturation = -2047
raw_fpaths = filePaths_by_stationName(timeID)
processed_data_dir = processed_data_dir(timeID)
output_fpath = processed_data_dir + output_folder
if not isdir(output_fpath):
mkdir(output_fpath)
log = logger()
log.set(output_fpath + '/log.txt', True)
log.take_stdout()
for station, fpaths in raw_fpaths.items():
print(station)
TBB_data = MultiFile_Dal1(fpaths)
for i, ant_name in enumerate(TBB_data.get_antenna_names()):
num_data_points = 0
num_saturated = 0
for block in range(initial_block, final_block):
data = TBB_data.get_data(block * block_size,
block_size,
def run_fitter(timeID,
output_folder,
pulse_input_folders,
guess_timings,
souces_to_fit,
guess_source_locations,
source_polarizations,
source_stations_to_exclude,
source_antennas_to_exclude,
bad_ants,
ref_station="CS002",
min_ant_amplitude=10,
num_itters=1000,
error_deviation=0.5E-9,
antenna_error=0.5E-9):
##### holdovers. These globals need to be fixed, so not global....
global station_locations, station_to_antenna_index_list, stations_with_fits, station_to_antenna_index_dict
global referance_station, station_order, sorted_antenna_names, min_antenna_amplitude, ant_locs, bad_antennas
global current_delays_guess, processed_data_folder
referance_station = ref_station
min_antenna_amplitude = min_ant_amplitude
bad_antennas = bad_ants
if referance_station in guess_timings:
ref_T = guess_timings[referance_station]
guess_timings = {
station: T - ref_T
for station, T in guess_timings.items()
if station != referance_station
}
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(
logging_folder +
"/log_out.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("timeID:", timeID)
print("date and time run:", time.strftime("%c"))
print("input folders:", pulse_input_folders)
print("source IDs to fit:", souces_to_fit)
print("guess locations:", guess_source_locations)
print("polarization to use:", source_polarizations)
print("source stations to exclude:", source_stations_to_exclude)
print("source antennas to exclude:", source_antennas_to_exclude)
print("bad antennas:", bad_ants)
print("referance station:", ref_station)
print("guess delays:", guess_timings)
print('pulse error:', error_deviation)
print('antenna error:', antenna_error)
print()
#### open data and data processing stuff ####
print("loading data")
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {
sname: MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
for sname, fpaths in raw_fpaths.items()
if sname in chain(guess_timings.keys(), [referance_station])
}
#### sort antennas and stations ####
station_order = list(
guess_timings.keys()) ## note this doesn't include reference station
sorted_antenna_names = []
station_to_antenna_index_dict = {}
ant_loc_dict = {}
for sname in station_order + [referance_station]:
first_index = len(sorted_antenna_names)
stat_data = raw_data_files[sname]
even_ant_names = stat_data.get_antenna_names()[::2]
even_ant_locs = stat_data.get_LOFAR_centered_positions()[::2]
sorted_antenna_names += even_ant_names
for ant_name, ant_loc in zip(even_ant_names, even_ant_locs):
ant_loc_dict[ant_name] = ant_loc
station_to_antenna_index_dict[sname] = (first_index,
len(sorted_antenna_names))
ant_locs = np.zeros((len(sorted_antenna_names), 3))
for i, ant_name in enumerate(sorted_antenna_names):
ant_locs[i] = ant_loc_dict[ant_name]
station_locations = {
sname: ant_locs[station_to_antenna_index_dict[sname][0]]
for sname in station_order + [referance_station]
}
station_to_antenna_index_list = [
station_to_antenna_index_dict[sname]
for sname in station_order + [referance_station]
]
#### sort the delays guess, and account for station locations ####
current_delays_guess = np.array(
[guess_timings[sname] for sname in station_order])
# original_delays = np.array( current_delays_guess )
#### open info from part 1 ####
input_manager = Part1_input_manager(pulse_input_folders)
#### first we fit the known sources ####
current_sources = []
# next_source = 0
for knownID in souces_to_fit:
source_ID, input_name = input_manager.known_source(knownID)
print("prep fitting:", source_ID)
location = guess_source_locations[source_ID]
## make source
source_to_add = source_object(source_ID, input_name, location,
source_stations_to_exclude[source_ID],
source_antennas_to_exclude[source_ID],
num_itters)
current_sources.append(source_to_add)
polarity = source_polarizations[source_ID]
source_to_add.prep_for_fitting(polarity, guess_timings)
fitter = stochastic_fitter_dt(current_sources)
all_delays = np.empty((num_itters, fitter.num_delays), dtype=np.double)
all_RMSs = np.empty(num_itters, dtype=np.double)
for i in range(num_itters):
all_delays[i, :], all_RMSs[i] = fitter.rerun(error_deviation,
antenna_error)
fitter.employ_result(current_sources)
print('run', i, 'RMS:', all_RMSs[i])
print()
print()
print("station timing errors:")
for i, sname in zip(range(fitter.num_delays), station_order):
print(sname, ":", np.std(all_delays[:, i]))
print()
print()
### get average X, Y, Z for each itteraion
ave_X = np.zeros(num_itters)
ave_Y = np.zeros(num_itters)
ave_Z = np.zeros(num_itters)
for source in current_sources:
ave_X += source.solutions[:, 0]
ave_Y += source.solutions[:, 1]
ave_Z += source.solutions[:, 2]
ave_X /= len(current_sources)
ave_Y /= len(current_sources)
ave_Z /= len(current_sources)
print("absolute location errors:")
print("X", np.std(ave_X), "Y", np.std(ave_Y), "Z", np.std(ave_Z))
print()
print()
print("relative location errors")
for source in current_sources:
source.solutions[:, 0] -= ave_X
source.solutions[:, 1] -= ave_Y
source.solutions[:, 2] -= ave_Z
print("source", source.ID)
print(" ", np.std(source.solutions[:, 0]),
np.std(source.solutions[:, 1]), np.std(source.solutions[:, 2]))
print()
print()
print("average RMS", np.average(all_RMSs), "std of RMS", np.std(all_RMSs))
def run_multiple_blocks(self,
output_folder,
initial_datapoint,
start_block,
blocks_per_run,
run_number,
skip_blocks_done=True,
print_to_screen=True):
processed_data_folder = processed_data_dir(self.timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
file_number = 0
while True:
fname = logging_folder + "/log_run_" + str(file_number) + ".txt"
if isfile(fname):
file_number += 1
else:
break
logger_function = logger()
logger_function.set(fname, print_to_screen)
logger_function.take_stdout()
logger_function.take_stderr()
#### TODO!#### improve log all options
logger_function("timeID:", self.timeID)
logger_function("date and time run:", time.strftime("%c"))
logger_function("output folder:", output_folder)
logger_function("block size:", self.block_size)
logger_function("initial data point:", initial_datapoint)
logger_function("first block:", start_block)
logger_function("blocks per run:", blocks_per_run)
logger_function("run number:", run_number)
logger_function("station delay file:", self.station_delays_fname)
logger_function("additional antenna delays file:",
self.additional_antenna_delays_fname)
logger_function("bad antennas file:", self.bad_antennas_fname)
logger_function("pol flips file:", self.pol_flips_fname)
logger_function("bounding box:", self.bounding_box)
logger_function("pulse length:", self.pulse_length)
logger_function("num antennas per station:",
self.num_antennas_per_station)
logger_function("stations excluded:", self.stations_to_exclude)
logger_function("prefered station:", self.prefered_station)
#### open files, save header if necisary ####
self.open_files(logger_function, True)
if run_number == 0:
header_outfile = h5py.File(data_dir + "/header.h5", "w")
self.save_header(header_outfile)
header_outfile.close()
#### run the algorithm!! ####
for block_index in range(run_number * blocks_per_run + start_block,
(run_number + 1) * blocks_per_run +
start_block):
out_fname = data_dir + "/block_" + str(block_index) + ".h5"
tmp_fname = data_dir + "/tmp_" + str(block_index) + ".h5"
if skip_blocks_done and isfile(out_fname):
logger_function("block:", block_index,
"already completed. Skipping")
continue
start_time = time.time()
block_outfile = h5py.File(tmp_fname, "w")
block_start = initial_datapoint + self.active_block_length * block_index
logger_function("starting processing block:", block_index, "at",
block_start)
self.process_block(block_start,
block_index,
block_outfile,
log_func=logger_function)
block_outfile.close()
rename(tmp_fname, out_fname)
logger_function("block done. took:", (time.time() - start_time),
's')
logger_function()
logger_function("done processing")
def save_EventByLoc(timeID, XYZT, station_timing_delays, pulse_index, output_folder, pulse_width=50, min_ant_amp=5, upsample_factor=0,
polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
processed_data_folder = processed_data_dir(timeID)
polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
additional_antenna_delays = read_antenna_delays( processed_data_folder + '/' + additional_antenna_delays )
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(logging_folder + "/log_out_"+str(pulse_index)+"_.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("saving traces to file", pulse_index)
print("location:", XYZT)
print("upsample_factor:", upsample_factor)
print()
print()
print("station timing offsets")
print(station_timing_delays)
print()
print()
print("pol flips")
print(polarization_flips)
print()
print()
print("bad antennas")
print(bad_antennas)
print()
print()
print("additional antenna delays")
print(additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays,
station_delay=station_timing_delays[sname]) for sname,fpaths in raw_fpaths.items() if sname in station_timing_delays}
data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_timing_delays}
trace_locator = getTrace_fromLoc( raw_data_files, data_filters )
print()
print()
print("data opened")
out_fname = data_dir + "/potSource_"+str(pulse_index)+".h5"
out_file = h5py.File(out_fname, "w")
for sname in station_timing_delays.keys():
TBB_file = raw_data_files[ sname ]
TBB_file.find_and_set_polarization_delay()
antenna_names = TBB_file.get_antenna_names()
h5_statGroup = out_file.create_group( sname )
print()
print(sname)
for even_ant_i in range(0,len(antenna_names),2):
even_ant_name = antenna_names[ even_ant_i ]
odd_ant_name = antenna_names[ even_ant_i+1 ]
starting_index, PolE_offset, throw, PolE_trace = trace_locator.get_trace_fromLoc(XYZT, even_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
throw, PolO_offset, throw, PolO_trace = trace_locator.get_trace_fromIndex(starting_index, odd_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
PolE_HE = np.abs(PolE_trace)
PolO_HE = np.abs(PolO_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(even_ant_name, (4, pulse_width ), dtype=np.double)
h5_Ant_dataset[0] = np.real(PolE_trace)
h5_Ant_dataset[1] = PolE_HE
h5_Ant_dataset[2] = np.real(PolO_trace)
h5_Ant_dataset[3] = PolO_HE
### note that peak time should NOT account for station timing offsets!
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs['PolE_timeOffset'] = -(PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset'] = -(PolO_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolE_timeOffset_CS'] = (PolE_offset - station_timing_delays[sname])
h5_Ant_dataset.attrs['PolO_timeOffset_CS'] = (PolO_offset - station_timing_delays[sname])
sample_time = 5.0E-9
if upsample_factor > 1:
PolE_HE = resample(PolE_HE, len(PolE_HE)*upsample_factor )
PolO_HE = resample(PolO_HE, len(PolO_HE)*upsample_factor )
sample_time /= upsample_factor
if np.max(PolE_HE)> min_ant_amp:
PolE_peak_finder = parabolic_fit( PolE_HE )
h5_Ant_dataset.attrs['PolE_peakTime'] = starting_index*5.0E-9 - (PolE_offset-station_timing_delays[sname]) + PolE_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(PolO_HE)> min_ant_amp:
PolO_peak_finder = parabolic_fit( PolO_HE )
h5_Ant_dataset.attrs['PolO_peakTime'] = starting_index*5.0E-9 - (PolO_offset-station_timing_delays[sname]) + PolO_peak_finder.peak_index*sample_time
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", even_ant_name, h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'])
out_file.close()
def run_fitter(timeID, output_folder, pulse_input_folders, guess_timings, souces_to_fit, guess_source_locations,
source_polarizations, source_stations_to_exclude, source_antennas_to_exclude, bad_ants,
X_deviations, Y_deviations, Z_deviations,
ref_station="CS002", min_ant_amplitude=10, max_stoch_loop_itters = 2000, min_itters_till_convergence = 100,
initial_jitter_width = 100000E-9, final_jitter_width = 1E-9, cooldown_fraction = 10.0, strong_cooldown_fraction = 100.0,
fitter = "dt"):
##### holdovers. These globals need to be fixed, so not global....
global station_locations, station_to_antenna_index_list, stations_with_fits, station_to_antenna_index_dict
global referance_station, station_order, sorted_antenna_names, min_antenna_amplitude, ant_locs, bad_antennas
global max_itters_per_loop, itters_till_convergence, min_jitter_width, max_jitter_width, cooldown, strong_cooldown
global current_delays_guess, processed_data_folder
referance_station = ref_station
min_antenna_amplitude = min_ant_amplitude
bad_antennas = bad_ants
max_itters_per_loop = max_stoch_loop_itters
itters_till_convergence = min_itters_till_convergence
max_jitter_width = initial_jitter_width
min_jitter_width = final_jitter_width
cooldown = cooldown_fraction
strong_cooldown = strong_cooldown_fraction
if referance_station in guess_timings:
ref_T = guess_timings[referance_station]
guess_timings = {station:T-ref_T for station,T in guess_timings.items() if station != referance_station}
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
#Setup logger and open initial data set
log = logger()
log.set(logging_folder + "/log_out.txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("timeID:", timeID)
print("date and time run:", time.strftime("%c") )
print("input folders:", pulse_input_folders)
print("source IDs to fit:", souces_to_fit)
print("guess locations:", guess_source_locations)
print("polarization to use:", source_polarizations)
print("source stations to exclude:", source_stations_to_exclude)
print("source antennas to exclude:", source_antennas_to_exclude)
print("bad antennas:", bad_ants)
print("referance station:", ref_station)
print("fitter type:", fitter)
print("guess delays:", guess_timings)
print()
print()
#### open data and data processing stuff ####
print("loading data")
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True) for sname,fpaths in raw_fpaths.items() if sname in chain(guess_timings.keys(), [referance_station]) }
#### sort antennas and stations ####
station_order = list(guess_timings.keys())## note this doesn't include reference station
sorted_antenna_names = []
station_to_antenna_index_dict = {}
ant_loc_dict = {}
for sname in station_order + [referance_station]:
first_index = len(sorted_antenna_names)
stat_data = raw_data_files[sname]
even_ant_names = stat_data.get_antenna_names()[::2]
even_ant_locs = stat_data.get_LOFAR_centered_positions()[::2]
sorted_antenna_names += even_ant_names
for ant_name, ant_loc in zip(even_ant_names,even_ant_locs):
ant_loc_dict[ant_name] = ant_loc
station_to_antenna_index_dict[sname] = (first_index, len(sorted_antenna_names))
ant_locs = np.zeros( (len(sorted_antenna_names), 3))
for i, ant_name in enumerate(sorted_antenna_names):
ant_locs[i] = ant_loc_dict[ant_name]
station_locations = {sname:ant_locs[station_to_antenna_index_dict[sname][0]] for sname in station_order + [referance_station]}
station_to_antenna_index_list = [station_to_antenna_index_dict[sname] for sname in station_order + [referance_station]]
#### sort the delays guess, and account for station locations ####
current_delays_guess = np.array([guess_timings[sname] for sname in station_order])
original_delays = np.array( current_delays_guess )
#### open info from part 1 ####
input_manager = Part1_input_manager( pulse_input_folders )
#### first we fit the known sources ####
current_sources = []
# next_source = 0
for knownID in souces_to_fit:
source_ID, input_name = input_manager.known_source( knownID )
print("prep fitting:", source_ID)
location = guess_source_locations[source_ID]
## make source
source_to_add = source_object(source_ID, input_name, location, source_stations_to_exclude[source_ID], source_antennas_to_exclude[source_ID] )
current_sources.append( source_to_add )
polarity = source_polarizations[source_ID]
source_to_add.prep_for_fitting(polarity)
print()
print("fitting known sources")
if fitter!='dt':
print("only dt")
quit()
first_source = current_sources[0]
original_XYZT = np.array( first_source.guess_XYZT )
if X_deviations is not None:
RMS_values = np.empty( len(X_deviations) )
first_source.guess_XYZT[:] = original_XYZT
for i in range(len(X_deviations)):
first_source.guess_XYZT[0] = original_XYZT[0] + X_deviations[i]
fitter = stochastic_fitter_dt(current_sources)
RMS_values[i] = fitter.RMS
print('X', X_deviations[i], fitter.RMS)
plt.plot(X_deviations, RMS_values)
plt.show()
if Y_deviations is not None:
RMS_values = np.empty( len(Y_deviations) )
first_source.guess_XYZT[:] = original_XYZT
for i in range(len(Y_deviations)):
first_source.guess_XYZT[1] = original_XYZT[1] + Y_deviations[i]
fitter = stochastic_fitter_dt(current_sources)
RMS_values[i] = fitter.RMS
print('Y', Y_deviations[i], fitter.RMS)
plt.plot(Y_deviations, RMS_values)
plt.show()
if Z_deviations is not None:
RMS_values = np.empty( len(Z_deviations) )
first_source.guess_XYZT[:] = original_XYZT
for i in range(len(Z_deviations)):
first_source.guess_XYZT[2] = original_XYZT[2] + Z_deviations[i]
fitter = stochastic_fitter_dt(current_sources)
RMS_values[i] = fitter.RMS
print('Z', Z_deviations[i], fitter.RMS)
plt.plot(Z_deviations, RMS_values)
plt.show()
def save_Pulse(timeID,
output_folder,
event_index,
pulse_time,
station_delays,
skip_stations=[],
window_width=7E-6,
pulse_width=50,
block_size=2**16,
min_ant_amp=5.0,
guess_flash_location=None,
referance_station="CS002",
polarization_flips="polarization_flips.txt",
bad_antennas="bad_antennas.txt",
additional_antenna_delays="ant_delays.txt"):
"""Desined to be used in conjuction with plot_multiple_data_all_stations.py. Given a pulse_time, and window_width, it saves a pulse on every antenna that is
centered on the largest peak in the window, with a width of pulse_width. station_delays doesn't need to be accurate, just should be the same as used in plot_multiple_data_all_stations.
Same for referance_station, and guess_flash_location. Note that this will save the pulse under an index, and will overwrite any other pulse saved under that index in the output folder"""
global index_range
if referance_station in station_delays:
ref_delay = station_delays[referance_station]
station_delays = {
sname: delay - ref_delay
for sname, delay in station_delays.items()
}
#### setup directory variables ####
processed_data_folder = processed_data_dir(timeID)
data_dir = processed_data_folder + "/" + output_folder
if not isdir(data_dir):
mkdir(data_dir)
logging_folder = data_dir + '/logs_and_plots'
if not isdir(logging_folder):
mkdir(logging_folder)
log = logger()
#Setup logger and open initial data set
log.set(logging_folder + "/log_event_" + str(event_index) +
".txt") ## TODo: save all output to a specific output folder
log.take_stderr()
log.take_stdout()
print("pulse time:", pulse_time)
print("window_width:", window_width)
print("pulse_width:", pulse_width)
print("skip stations:", skip_stations)
print("guess_flash_location:", guess_flash_location)
print("input delays:")
print(station_delays)
print()
##station data
print()
print("opening station data")
polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' +
bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder +
'/' +
additional_antenna_delays)
raw_fpaths = filePaths_by_stationName(timeID)
raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
for sname in station_delays.keys()}
data_filters = {
sname: window_and_filter(timeID=timeID, sname=sname)
for sname in station_delays.keys()
}
trace_locator = getTrace_fromLoc(
raw_data_files, data_filters,
{sname: 0.0
for sname in station_delays.keys()})
if guess_flash_location is not None:
guess_flash_location = np.array(guess_flash_location)
ref_stat_file = raw_data_files[referance_station]
ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
ref_delay = np.linalg.norm(
ant_loc - guess_flash_location[:3]
) / v_air - ref_stat_file.get_nominal_sample_number() * 5.0E-9
for sname, data_file in raw_data_files.items():
if sname not in station_delays:
station_delays[sname] = 0.0
data_file = raw_data_files[sname]
ant_loc = data_file.get_LOFAR_centered_positions()[0]
station_delays[sname] += (
np.linalg.norm(ant_loc - guess_flash_location[:3]) / v_air -
ref_delay)
station_delays[sname] -= data_file.get_nominal_sample_number(
) * 5.0E-9
print()
print("used apparent delays:")
print(station_delays)
print()
out_fname = data_dir + "/potSource_" + str(event_index) + ".h5"
out_file = h5py.File(out_fname, "w")
half_pulse_width = int(0.5 * pulse_width)
window_width_samples = int(window_width / 5.0E-9)
for sname in station_delays.keys():
if (not np.isfinite(station_delays[sname])) or sname in skip_stations:
continue
h5_statGroup = out_file.create_group(sname)
antenna_names = raw_data_files[sname].get_antenna_names()
data_arrival_index = int(
(pulse_time + station_delays[sname] - window_width * 0.5) / 5.0E-9)
print()
print(sname)
for even_ant_i in range(0, len(antenna_names), 2):
#### get window and find peak ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex(
data_arrival_index, antenna_names[even_ant_i],
window_width_samples)
throw, odd_total_time_offset, throw, odd_trace = trace_locator.get_trace_fromIndex(
data_arrival_index, antenna_names[even_ant_i + 1],
window_width_samples)
even_HE = np.abs(even_trace)
even_good = False
if np.max(even_HE) > min_ant_amp:
even_good = True
even_para_fit = parabolic_fit(even_HE)
odd_HE = np.abs(odd_trace)
odd_good = False
if np.max(odd_HE) > min_ant_amp:
odd_good = True
odd_para_fit = parabolic_fit(odd_HE)
if (not even_good) and (not odd_good):
continue
elif even_good and (not odd_good):
center_index = int(even_para_fit.peak_index)
elif (not even_good) and odd_good:
center_index = int(odd_para_fit.peak_index)
else: ## both good
even_amp = even_HE[int(even_para_fit.peak_index)]
odd_amp = odd_HE[int(odd_para_fit.peak_index)]
if even_amp > odd_amp:
center_index = int(even_para_fit.peak_index)
else:
center_index = int(odd_para_fit.peak_index)
#### now get data centered on peak. re-get data, as peak may be near edge ####
throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex(
data_arrival_index + center_index - half_pulse_width,
antenna_names[even_ant_i], half_pulse_width * 2)
throw, odd_total_time_offset, throw, odd_trace = trace_locator.get_trace_fromIndex(
data_arrival_index + center_index - half_pulse_width,
antenna_names[even_ant_i + 1], half_pulse_width * 2)
# even_trace = even_trace[center_index-half_pulse_width:center_index+half_pulse_width]
# odd_trace = odd_trace[center_index-half_pulse_width:center_index+half_pulse_width]
even_HE = np.abs(even_trace)
odd_HE = np.abs(odd_trace)
h5_Ant_dataset = h5_statGroup.create_dataset(
antenna_names[even_ant_i], (4, half_pulse_width * 2),
dtype=np.double)
h5_Ant_dataset[0] = np.real(even_trace)
h5_Ant_dataset[1] = even_HE
h5_Ant_dataset[2] = np.real(odd_trace)
h5_Ant_dataset[3] = odd_HE
#WARNING: not sure this is correct. even_total_time_offset includes station delay
starting_index = data_arrival_index + center_index - half_pulse_width
h5_Ant_dataset.attrs['starting_index'] = starting_index
h5_Ant_dataset.attrs[
'PolE_timeOffset'] = -even_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
h5_Ant_dataset.attrs[
'PolO_timeOffset'] = -odd_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
if np.max(even_HE) > min_ant_amp:
# even_HE = resample(even_HE, len(even_HE)*8 )
# even_para_fit = parabolic_fit( even_HE )
# h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - even_total_time_offset
even_para_fit = parabolic_fit(even_HE)
h5_Ant_dataset.attrs['PolE_peakTime'] = (
even_para_fit.peak_index +
starting_index) * 5.0E-9 - even_total_time_offset
else:
h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
if np.max(odd_HE) > min_ant_amp:
# odd_HE = resample(odd_HE, len(odd_HE)*8 )
# odd_para_fit = parabolic_fit( odd_HE )
# h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - odd_total_time_offset
odd_para_fit = parabolic_fit(odd_HE)
h5_Ant_dataset.attrs['PolO_peakTime'] = (
odd_para_fit.peak_index +
starting_index) * 5.0E-9 - odd_total_time_offset
else:
h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
print(" ", antenna_names[even_ant_i],
(data_arrival_index + center_index) * 5.0E-9 -
station_delays[sname], h5_Ant_dataset.attrs['PolE_peakTime'],
h5_Ant_dataset.attrs['PolO_peakTime'], starting_index)
index_range.append(starting_index)
out_file.close()
index_range = [min(index_range), max(index_range)]
print("rand of indexes: " + str(index_range))