コード例 #1
0
def planewave_fits(timeID,
                   station,
                   polarization,
                   initial_block,
                   number_of_blocks,
                   pulses_per_block=10,
                   pulse_length=50,
                   min_amplitude=50,
                   upsample_factor=0,
                   min_num_antennas=4,
                   polarization_flips="polarization_flips.txt",
                   bad_antennas="bad_antennas.txt",
                   additional_antenna_delays="ant_delays.txt",
                   max_num_planewaves=np.inf,
                   positive_saturation=2046,
                   negative_saturation=-2047,
                   saturation_post_removal_length=50,
                   saturation_half_hann_length=50,
                   verbose=True):
    """ wrapper around 'planewave_fitter' only present for backwards compatiblility. Returns RMS, zenithal and azimuthal fit values, as three arrays in that order"""

    processed_data_folder = processed_data_dir(timeID)
    polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
                                                polarization_flips)
    bad_antennas = read_bad_antennas(processed_data_folder + '/' +
                                     bad_antennas)
    additional_antenna_delays = read_antenna_delays(processed_data_folder +
                                                    '/' +
                                                    additional_antenna_delays)

    raw_fpaths = filePaths_by_stationName(timeID)
    TBB_data = MultiFile_Dal1(raw_fpaths[station],
                              polarization_flips=polarization_flips,
                              bad_antennas=bad_antennas,
                              additional_ant_delays=additional_antenna_delays)

    fitter = planewave_fitter(
        TBB_data=TBB_data,
        polarization=polarization,
        initial_block=initial_block,
        number_of_blocks=number_of_blocks,
        pulses_per_block=pulses_per_block,
        pulse_length=pulse_length,
        min_amplitude=min_amplitude,
        upsample_factor=upsample_factor,
        min_num_antennas=min_num_antennas,
        max_num_planewaves=max_num_planewaves,
        verbose=verbose,  ## doesn't do anything anyway
        positive_saturation=positive_saturation,
        negative_saturation=negative_saturation,
        saturation_post_removal_length=saturation_post_removal_length,
        saturation_half_hann_length=saturation_half_hann_length)

    RMSs, Zeniths, Azimuths, throw = fitter.go_fit()
    return RMSs, Zeniths, Azimuths
コード例 #2
0
def save_Pulse(timeID, output_folder, event_index, pulse_time, station_delays, skip_stations=[], window_width=7E-6, pulse_width=50, block_size=2**16, 
               min_ant_amp=5.0, guess_flash_location=None, referance_station="CS002", polarization_flips="polarization_flips.txt", 
               bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
    """Desined to be used in conjuction with plot_multiple_data_all_stations.py. Given a pulse_time, and window_width, it saves a pulse on every antenna that is 
    centered on the largest peak in the window, with a width of pulse_width. station_delays doesn't need to be accurate, just should be the same as used in plot_multiple_data_all_stations.
    Same for referance_station, and guess_flash_location. Note that this will save the pulse under an index, and will overwrite any other pulse saved under that index in the output folder"""
    
    
    if referance_station in station_delays:
        ref_delay = station_delays[referance_station]
        station_delays = {sname:delay-ref_delay for sname,delay in station_delays.items()}
    
    #### setup directory variables ####
    processed_data_folder = processed_data_dir(timeID)
    
    data_dir = processed_data_folder + "/" + output_folder
    if not isdir(data_dir):
        mkdir(data_dir)
        
    logging_folder = data_dir + '/logs_and_plots'
    if not isdir(logging_folder):
        mkdir(logging_folder)

    #Setup logger and open initial data set
    log.set(logging_folder + "/log_event_"+str(event_index)+".txt") ## TODo: save all output to a specific output folder
    log.take_stderr()
    log.take_stdout()
    
    print("pulse time:", pulse_time)
    print("window_width:", window_width)
    print("pulse_width:", pulse_width)
    print("skip stations:", skip_stations)
    print("guess_flash_location:", guess_flash_location)
    print("input delays:")
    print( station_delays)
    print()
    
        ##station data
    print()
    print("opening station data")
    
    polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
    bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
    additional_antenna_delays = read_antenna_delays(  processed_data_folder + '/' + additional_antenna_delays )
    
        
    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
                      for sname in station_delays.keys()}
    
    data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_delays.keys()}
    
    trace_locator = getTrace_fromLoc( raw_data_files, data_filters, {sname:0.0 for sname in station_delays.keys()} )
    
    
    if guess_flash_location is not None:
        guess_flash_location = np.array(guess_flash_location)
        
        ref_stat_file = raw_data_files[ referance_station ]
        ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
        ref_delay = np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9
        
        for sname, data_file in raw_data_files.items():
            if sname not in station_delays:
                station_delays[sname] = 0.0
            
            data_file = raw_data_files[sname]
            ant_loc = data_file.get_LOFAR_centered_positions()[0]
            station_delays[sname] += (np.linalg.norm(ant_loc-guess_flash_location[:3])/v_air - ref_delay) 
            station_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9
    
    print()
    print("used apparent delays:")
    print(station_delays)
    print()
    
    
    out_fname = data_dir + "/potSource_"+str(event_index)+".h5"
    out_file = h5py.File(out_fname, "w")
    
    half_pulse_width = int(0.5*pulse_width)
    window_width_samples = int(window_width/5.0E-9)
    for sname in station_delays.keys():
        if (not np.isfinite(station_delays[sname])) or sname in skip_stations:
            continue
        
        h5_statGroup = out_file.create_group( sname )
        
        antenna_names = raw_data_files[sname].get_antenna_names()
        
        data_arrival_index = int((pulse_time + station_delays[sname] - window_width*0.5)/5.0E-9)
        
        print()
        print(sname)
        
        for even_ant_i in range(0, len(antenna_names), 2):
            
            #### get window and find peak ####
            throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i], window_width_samples)
            throw, odd_total_time_offset, throw, odd_trace = trace_locator.get_trace_fromIndex( data_arrival_index, antenna_names[even_ant_i+1], window_width_samples)

            even_HE = np.abs(even_trace)
            even_good = False
            if np.max( even_HE ) > min_ant_amp:
                even_good = True
                even_para_fit = parabolic_fit( even_HE )
            
            odd_HE = np.abs(odd_trace)
            odd_good = False
            if np.max( odd_HE ) > min_ant_amp:
                odd_good = True
                odd_para_fit = parabolic_fit( odd_HE )
                
            if (not even_good) and (not odd_good):
                continue
            elif even_good and (not odd_good):
                center_index = int(even_para_fit.peak_index)
            elif (not even_good) and odd_good:
                center_index = int(odd_para_fit.peak_index)
            else: ## both good
                even_amp = even_HE[ int(even_para_fit.peak_index) ]
                odd_amp = odd_HE[ int(odd_para_fit.peak_index) ]
                if even_amp > odd_amp:
                    center_index = int(even_para_fit.peak_index)
                else:
                    center_index = int(odd_para_fit.peak_index)

            
            #### now get data centered on peak. re-get data, as peak may be near edge ####
            throw, even_total_time_offset, throw, even_trace = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i], half_pulse_width*2)
            throw, odd_total_time_offset , throw, odd_trace  = trace_locator.get_trace_fromIndex( data_arrival_index+center_index-half_pulse_width, antenna_names[even_ant_i+1], half_pulse_width*2)
                  
#            even_trace = even_trace[center_index-half_pulse_width:center_index+half_pulse_width]
#            odd_trace = odd_trace[center_index-half_pulse_width:center_index+half_pulse_width]
            even_HE = np.abs(even_trace)
            odd_HE = np.abs(odd_trace)
                
            h5_Ant_dataset = h5_statGroup.create_dataset(antenna_names[even_ant_i], (4, half_pulse_width*2), dtype=np.double)
            h5_Ant_dataset[0] = np.real( even_trace )
            h5_Ant_dataset[1] = even_HE
            h5_Ant_dataset[2] = np.real( odd_trace )
            h5_Ant_dataset[3] = odd_HE
                        
            WARNING: not sure this is correct. even_total_time_offset includes station delay
            starting_index = data_arrival_index+center_index-half_pulse_width
            h5_Ant_dataset.attrs['starting_index'] = starting_index
            h5_Ant_dataset.attrs['PolE_timeOffset'] = -even_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
            h5_Ant_dataset.attrs['PolO_timeOffset'] = -odd_total_time_offset ## NOTE: due to historical reasons, there is a sign flip here
               
            if np.max(even_HE) > min_ant_amp:
#                even_HE = resample(even_HE, len(even_HE)*8 )
#                even_para_fit = parabolic_fit( even_HE )
#                h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - even_total_time_offset
                even_para_fit = parabolic_fit( even_HE )
                h5_Ant_dataset.attrs['PolE_peakTime'] = (even_para_fit.peak_index + starting_index)*5.0E-9 - even_total_time_offset
            else:
                h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
                 
            if np.max(odd_HE) > min_ant_amp:
#                odd_HE = resample(odd_HE, len(odd_HE)*8 )
#                odd_para_fit = parabolic_fit( odd_HE )
#                h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index/8.0 + starting_index)*5.0E-9 - odd_total_time_offset
                odd_para_fit = parabolic_fit( odd_HE )
                h5_Ant_dataset.attrs['PolO_peakTime'] = (odd_para_fit.peak_index + starting_index)*5.0E-9 - odd_total_time_offset
            else:
                h5_Ant_dataset.attrs['PolO_peakTime'] = np.nan
                
            
            print("   ", antenna_names[even_ant_i], (data_arrival_index+center_index)*5.0E-9 - station_delays[sname], h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'] )
            
    out_file.close()
                
コード例 #3
0
def plot_station(event_ID, sname_to_plot, plot_bad_antennas,
               timeID, output_folder, pulse_input_folders, guess_timings, sources_to_fit, guess_source_locations,
               source_polarizations, source_stations_to_exclude, source_antennas_to_exclude, bad_ants,
               antennas_to_recalibrate={}, min_ant_amplitude=10, ref_station="CS002"):
    
    processed_data_folder = processed_data_dir( timeID )
    file_manager = input_manager( processed_data_folder, pulse_input_folders )
    throw, input_Fname = file_manager.known_source( event_ID )
    
    
    data_file = h5py.File(input_Fname, "r")
    stat_group = data_file[ sname_to_plot ]
    
    fpaths = filePaths_by_stationName( timeID )[ sname_to_plot ]
    station_file = MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
    
    source_XYZT = guess_source_locations[event_ID]
    source_polarization = source_polarizations[event_ID]
    antennas_to_exclude = bad_ants + source_antennas_to_exclude[event_ID]
    
    ref_station_delay = 0.0
    if ref_station in guess_timings:
        ref_station_delay = guess_timings[ref_station]
        
    stat_delay = 0.0
    if sname_to_plot != ref_station:
        stat_delay = guess_timings[sname_to_plot] - ref_station_delay
    
    even_HEs = []
    odd_HEs = []
    even_sig = []
    odd_sig = []
    even_good = []
    odd_good = []
    even_offet = []
    odd_offset = []
    even_peakT_offset = []
    odd_peakT_offset = []
    even_ant_names = []
    odd_ant_names = []
    
    peak_amp = 0.0
    ## first we open the data
    even_SSqE = 0.0
    odd_SSqE = 0.0
    even_N = 0
    odd_N = 0
    for ant_name, ant_loc in zip(station_file.get_antenna_names(),station_file.get_LOFAR_centered_positions()):
        if ant_name not in stat_group:
            continue
        
        even_ant_name = ant_name
        odd_ant_name = even_antName_to_odd( ant_name )
        
        ant_dataset = stat_group[ant_name]
        even_trace = np.array( ant_dataset[0] )
        even_HE = np.array( ant_dataset[1] )
        odd_trace = np.array( ant_dataset[2] )
        odd_HE = np.array( ant_dataset[3] )
        
        even_amp = np.max(even_HE)
        odd_amp = np.max(odd_HE)
        
        M = max(even_amp, odd_amp)
        if M>peak_amp:
            peak_amp = M
        
        travel_delay = np.sqrt( (ant_loc[0]-source_XYZT[0])**2 + (ant_loc[1]-source_XYZT[1])**2 + (ant_loc[2]-source_XYZT[2])**2 )/v_air
        travel_delay_odd = travel_delay
        
        if source_polarization==3  and len(source_XYZT)==8:
            travel_delay_odd = np.sqrt( (ant_loc[0]-source_XYZT[4])**2 + (ant_loc[1]-source_XYZT[5])**2 + (ant_loc[2]-source_XYZT[6])**2 )/v_air
        
        even_time =  - travel_delay - stat_delay -source_XYZT[3]
        odd_time =  - travel_delay_odd - stat_delay
        
        if even_ant_name in antennas_to_recalibrate:
            even_time -= antennas_to_recalibrate[even_ant_name]
            
        if odd_ant_name in antennas_to_recalibrate:
            odd_time -= antennas_to_recalibrate[odd_ant_name]  
            
        if source_polarization==0 or source_polarization==1 or len(source_XYZT)==4:
            odd_time -= source_XYZT[3]
        elif (source_polarization==2 or source_polarization==3) and len(source_XYZT)==5:
            odd_time -= source_XYZT[4]
        elif (source_polarization==2 or source_polarization==3)  and len(source_XYZT)==8:
            odd_time -= source_XYZT[7]
            
        even_peak_time = ant_dataset.attrs["PolE_peakTime"] + even_time
        odd_peak_time = ant_dataset.attrs["PolO_peakTime"] + odd_time
            
        even_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolE_timeOffset_CS']
        odd_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolO_timeOffset_CS']
        
        
        even_is_good = even_amp>min_ant_amplitude
        if not even_is_good:
            print( even_ant_name, ": amp too low" )
        else:
            even_is_good = not (even_ant_name in antennas_to_exclude)
            if not even_is_good:
                print( even_ant_name, ": excluded" )
            else:
                print( even_ant_name, ": good" )
            
        odd_is_good  = odd_amp>min_ant_amplitude
        if not odd_is_good:
            print( odd_ant_name, ": amp too low" )
        else:
            odd_is_good = not (odd_ant_name in antennas_to_exclude)
            if not odd_is_good:
                print( odd_ant_name, ": excluded" )
            else:
                print( odd_ant_name, ": good" )
                
            
        even_HEs.append( even_HE )
        odd_HEs.append( odd_HE )
        even_sig.append( even_trace )
        odd_sig.append( odd_trace )
        even_good.append( even_is_good )
        odd_good.append( odd_is_good )
        even_offet.append( even_time )
        odd_offset.append( odd_time )
        even_ant_names.append( even_ant_name )
        odd_ant_names.append( odd_ant_name )
        even_peakT_offset.append( even_peak_time )
        odd_peakT_offset.append( odd_peak_time )
        
        if even_is_good:
            even_SSqE += even_peak_time*even_peak_time
            even_N += 1
            
        if odd_is_good:
            odd_SSqE += odd_peak_time*odd_peak_time
            odd_N += 1
    if even_N > 0:
        print('even RMS:', np.sqrt(even_SSqE/even_N),end=' ')
    if odd_N > 0:
        print('odd RMS:', np.sqrt(odd_SSqE/odd_N), end='')
    print()
        
    current_offset = 0
    for pair_i in range(len(even_HEs)):
        
        even_T = np.arange( len(even_HEs[pair_i]) )*5.0E-9 + even_offet[pair_i]
        odd_T = np.arange( len(odd_HEs[pair_i]) )*5.0E-9 + odd_offset[pair_i]
        
        if even_good[pair_i] or plot_bad_antennas:
            plt.plot(even_T, even_HEs[pair_i]/peak_amp + current_offset, 'g')
            plt.plot(even_T, even_sig[pair_i]/peak_amp + current_offset, 'g')
            
        if even_good[pair_i]:
            plt.plot( (even_peakT_offset[pair_i],even_peakT_offset[pair_i]), (current_offset,current_offset+1), 'g' )
            
        if odd_good[pair_i] or plot_bad_antennas:
            plt.plot(odd_T, odd_HEs[pair_i]/peak_amp + current_offset, 'm')
            plt.plot(odd_T, odd_sig[pair_i]/peak_amp + current_offset, 'm')
            
        if odd_good[pair_i]:
            plt.plot( (odd_peakT_offset[pair_i],odd_peakT_offset[pair_i]), (current_offset,current_offset+1), 'm' )
        
        even_str = even_ant_names[pair_i] + " "
        odd_str = odd_ant_names[pair_i] + " "
        
        if even_good[pair_i]:
            even_str += ': good'
        else:
            even_str += ': bad'
        
        if odd_good[pair_i]:
            odd_str += ': good'
        else:
            odd_str += ': bad'
            
        plt.annotate(odd_str,  xy=(odd_T[0], current_offset+0.3), c='m')
        plt.annotate(even_str, xy=(even_T[0], current_offset+0.6), c='g')
        
        current_offset += 2
        
    plt.axvline(x=0)
    plt.show()
            
        
        
        
    
コード例 #4
0
    block_size = 2**16

    processed_data_dir = processed_data_dir(timeID)

    output_fpath = processed_data_dir + output_folder
    if not isdir(output_fpath):
        mkdir(output_fpath)

    #### get paths to raw data by station ####
    raw_fpaths = filePaths_by_stationName(timeID)
    for station in raw_fpaths.keys():

        print("station:", station)

        #### open the data for this station ####
        TBB_data = MultiFile_Dal1(raw_fpaths[station])

        num_antennas = len(TBB_data.get_antenna_names())
        num_blocks = int(
            np.min(TBB_data.get_nominal_data_lengths()) / block_size
        )  ### note that this throws away the last partial data block

        plt.figure()
        for antenna_i in range(num_antennas):
            #            if antenna_i == 1:
            #                break
            print("  antenna:", antenna_i, "/", num_antennas)

            data = np.zeros(num_blocks, dtype=float)

            for block_i in range(num_blocks):
コード例 #5
0
import numpy as np
import matplotlib.pyplot as plt

from LoLIM.utilities import processed_data_dir
from LoLIM.IO.raw_tbb_IO import MultiFile_Dal1, filePaths_by_stationName
from LoLIM.signal_processing import remove_saturation
from LoLIM.findRFI import window_and_filter
from LoLIM.read_pulse_data import read_station_delays, read_antenna_pol_flips, read_bad_antennas, read_antenna_delays

## these lines are anachronistic and should be fixed at some point
from LoLIM import utilities
utilities.default_raw_data_loc = "/exp_app2/appexp1/public/raw_data"
utilities.default_processed_data_loc = "/home/brian/processed_files"

if __name__ == "__main__":
    timeID = 'D20170929T202255.000Z'

    raw_fpaths = filePaths_by_stationName(timeID)

    for station, fpaths in raw_fpaths.items():
        TBB_data = MultiFile_Dal1(fpaths)
        locs = TBB_data.get_LOFAR_centered_positions()
        names = TBB_data.get_antenna_names()

        print(station)
        for even_ant_i in range(0, len(locs), 2):
            name = names[even_ant_i]
            loc = locs[even_ant_i]
            print(name, loc)
        print()
コード例 #6
0
def plot_one_station_allData(timeID,
                             input_file_name,
                             source_XYZT,
                             known_station_delays,
                             station,
                             referance_station="CS002",
                             min_antenna_amplitude=10):
    """Plot all pulses on all stations in an autocorrelator file. polarization should be 0 or 1. station should be station name """

    if referance_station not in known_station_delays:
        known_station_delays[referance_station] = 0
    known_station_delays = {
        station: T - known_station_delays[referance_station]
        for station, T in known_station_delays.items()
    }

    data_file = h5py.File(input_file_name, "r")

    station_names = list(data_file.keys())
    known_station_delays = {
        sname:
        known_station_delays[sname] if sname in known_station_delays else 0.0
        for sname in station_names
    }

    raw_fpaths = filePaths_by_stationName(timeID)
    stat_group = data_file[station]
    stat_data = MultiFile_Dal1(raw_fpaths[station],
                               force_metadata_ant_pos=True)

    current_amp = 0.0
    for ant_name, ant_loc in zip(stat_data.get_antenna_names(),
                                 stat_data.get_LOFAR_centered_positions()):
        if ant_name not in stat_group:
            continue

        ant_dataset = stat_group[ant_name]
        even_trace = np.array(ant_dataset[0])
        even_HE = np.array(ant_dataset[1])
        odd_trace = np.array(ant_dataset[2])
        odd_HE = np.array(ant_dataset[3])
        trace_amp = max(np.max(even_HE), np.max(odd_HE))

        if trace_amp < min_antenna_amplitude:
            continue

        travel_delay = np.sqrt((ant_loc[0] - source_XYZT[0])**2 +
                               (ant_loc[1] - source_XYZT[1])**2 +
                               (ant_loc[2] - source_XYZT[2])**2) / v_air
        total_correction = travel_delay + source_XYZT[
            3] + known_station_delays[station]

        trace_time = np.arange(len(even_trace)) * 5.0E-9 + ant_dataset.attrs[
            'starting_index'] * 5.0E-9 - total_correction
        even_trace_time = trace_time + ant_dataset.attrs['PolE_timeOffset']
        plt.plot(even_trace_time, even_trace + current_amp, 'g')
        plt.plot(even_trace_time, even_HE + current_amp, 'g')

        odd_trace_time = trace_time + ant_dataset.attrs['PolO_timeOffset']
        plt.plot(odd_trace_time, odd_trace + current_amp, 'm')
        plt.plot(odd_trace_time, odd_HE + current_amp, 'm')

        plt.annotate(ant_name, xy=(0.0, current_amp + 0.5 * trace_amp))

        current_amp += trace_amp * 2
    plt.axvline(x=0)
    plt.show()
コード例 #7
0
def plot_stations(timeID,
                  polarization,
                  input_file_name,
                  source_XYZT,
                  known_station_delays,
                  stations="all",
                  referance_station="CS002",
                  min_antenna_amplitude=10,
                  skip_stations=[],
                  antennas_to_exclude=[],
                  seperation_factor=1,
                  plot_real=False,
                  plot_peak_time=True):
    """Plot all pulses on all stations in an autocorrelator file. polarization should be 0 or 1. stations should be "all", "RS", or "CS" """

    if polarization == 0:
        dataset_index = 1  ##1 for even HE, 3 for odd HE
        peaktime_pol = "PolE_peakTime"  ## even or odd peak times
        antOffset_pol = 'PolE_timeOffset'
    elif polarization == 1:
        dataset_index = 3  ##1 3 for odd HE
        peaktime_pol = "PolO_peakTime"  ## even or odd peak times
        antOffset_pol = 'PolO_timeOffset'

    if referance_station not in known_station_delays:
        known_station_delays[referance_station] = 0
    known_station_delays = {
        station: T - known_station_delays[referance_station]
        for station, T in known_station_delays.items()
    }

    data_file = h5py.File(input_file_name, "r")

    station_names = [
        sname for sname in data_file.keys() if sname not in skip_stations
    ]
    known_station_delays = {
        sname:
        known_station_delays[sname] if sname in known_station_delays else 0.0
        for sname in station_names
    }

    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {
        sname: MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True)
        for sname in station_names
    }

    stat_i = 0
    for (sname, stat_group) in data_file.items():
        if (stations != "all"
                and stations != sname[:2]) or sname in skip_stations:
            continue

        stat_data = raw_data_files[sname]
        ant_loc_dict = {
            ant_name: ant_loc
            for ant_name, ant_loc in zip(
                stat_data.get_antenna_names(),
                stat_data.get_LOFAR_centered_positions())
        }

        trace_max = 0.0
        for ant_name, ant_dataset in stat_group.items():
            trace = np.array(ant_dataset[dataset_index])
            if np.max(trace) > trace_max:
                trace_max = np.max(trace)

        trace_max *= 2
        step = stat_i * seperation_factor

        for ant_name, ant_dataset in stat_group.items():

            trace = np.array(ant_dataset[dataset_index])
            if np.max(
                    trace
            ) < min_antenna_amplitude or ant_name in antennas_to_exclude:
                continue

            ant_loc = ant_loc_dict[ant_name]
            travel_delay = np.sqrt((ant_loc[0] - source_XYZT[0])**2 +
                                   (ant_loc[1] - source_XYZT[1])**2 +
                                   (ant_loc[2] - source_XYZT[2])**2) / v_air
            total_correction = travel_delay + source_XYZT[
                3] + known_station_delays[sname]

            peak_time = ant_dataset.attrs[peaktime_pol] - total_correction
            color = None
            if plot_peak_time:
                p = plt.plot((peak_time, peak_time), (step, step + 0.5))
                color = p[0].get_color()

            trace_time = np.arange(len(trace)) * 5.0E-9 + ant_dataset.attrs[
                'starting_index'] * 5.0E-9 + ant_dataset.attrs[
                    antOffset_pol] - total_correction
            plt.plot(trace_time, trace / trace_max + step, color=color)

            if plot_real:
                rt = np.array(ant_dataset[dataset_index - 1])
                plt.plot(trace_time, rt / trace_max + step, color=color)

        stat_i += 1

        plt.annotate(sname, xy=(0.0, step + 0.5))
    plt.axvline(x=0)
    plt.show()
コード例 #8
0
    fpaths = filePaths_by_stationName(timeID)
    polarization_flips = processed_data_folder + '/' + "polarization_flips.txt"
    bad_antennas = processed_data_folder + '/' + "bad_antennas.txt"
    # additional_antenna_delays = processed_data_folder + '/' + "ant_delays.txt"
    additional_antenna_delays = None

    station_log = logger()
    station_log.take_stdout()

    #### open the station

    print("processing", station)
    station_log.set(output_fpath + '/' + station + '_log.txt')

    TBB_data = MultiFile_Dal1(fpaths[station],
                              polarization_flips=polarization_flips,
                              bad_antennas=bad_antennas,
                              additional_ant_delays=additional_antenna_delays)

    if TBB_data.needs_metadata():
        print(station, "does not have delay metadata. This will not work.")

    ant_names = TBB_data.get_antenna_names()
    file_delays = TBB_data.get_timing_callibration_delays(
        force_file_delays=True)
    print()
    print("file delays")
    for name, delay in zip(ant_names, file_delays):
        print(name, delay)

    get_station_history(station, history_folder)
    revisions = get_ordered_revisions(
コード例 #9
0
def planewave_fits(timeID, station, polarization, initial_block, number_of_blocks, pulses_per_block=10, pulse_length=50, min_amplitude=50, upsample_factor=0, min_num_antennas=4,
                    polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", max_num_planewaves=np.inf,
                    positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 50, verbose=True):
    
    left_pulse_length = int(pulse_length/2)
    right_pulse_length = pulse_length-left_pulse_length
    
    processed_data_folder = processed_data_dir(timeID)
    polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
    bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
    additional_antenna_delays = read_antenna_delays(  processed_data_folder + '/' + additional_antenna_delays )
    
    raw_fpaths = filePaths_by_stationName(timeID)
    TBB_data = MultiFile_Dal1( raw_fpaths[station], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays )
    ant_names = TBB_data.get_antenna_names()
    antenna_locations = TBB_data.get_LOFAR_centered_positions()
    antenna_delays = TBB_data.get_timing_callibration_delays()
    num_antenna_pairs = int( len( ant_names )/2 ) 
    
    if num_antenna_pairs < min_num_antennas:
        return np.array([]), np.array([]), np.array([])

    RFI_filter = window_and_filter(timeID=timeID, sname=station)
    block_size = RFI_filter.blocksize
    
    out_RMS = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
    out_zenith = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
    out_azimuth = np.empty( number_of_blocks*pulses_per_block, dtype=np.double )
    N = 0
    data = np.empty( (num_antenna_pairs,block_size), dtype=np.double )
    for block in range(initial_block, initial_block+number_of_blocks):
        if N >= max_num_planewaves:
            break
        
        #### open and filter data
        for pair_i in range(num_antenna_pairs):
            ant_i = pair_i*2 + polarization
            
            data[pair_i,:] = TBB_data.get_data(block*block_size, block_size, antenna_index=ant_i)
            remove_saturation(data[pair_i,:], positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
            np.abs( RFI_filter.filter( data[pair_i,:] ), out=data[pair_i,:])
    
    
        #### loop over finding planewaves
        i = 0
        while i < pulses_per_block:
            if N >= max_num_planewaves:
                break
            
            pulse_location = 0
            pulse_amplitude = 0
            
            ## find highest peak
            for pair_i, HE in enumerate(data):
                loc = np.argmax( HE )
                amp = HE[ loc ]
                
                if amp > pulse_amplitude:
                    pulse_amplitude = amp
                    pulse_location = loc
                    
            ## check if is strong enough
            if pulse_amplitude < min_amplitude:
                break
            
            ## get 
            fitter = locatefier()
            for pair_i, HE in enumerate(data):
                ant_i = pair_i*2 + polarization
                
                signal = np.array( HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] )
                if num_double_zeros(signal, threshold=0.1) == 0:
                    
                    sample_time = 5.0E-9
                    if upsample_factor > 1:
                        sample_time /= upsample_factor
                        signal = resample(signal, len(signal)*upsample_factor )
                        
                    peak_finder = parabolic_fit( signal )
                    peak_time = peak_finder.peak_index*sample_time - antenna_delays[ant_i]
                    fitter.add_antenna(peak_time, antenna_locations[ant_i])
                    
                HE[ pulse_location-left_pulse_length : pulse_location+right_pulse_length] = 0.0
            
            if fitter.num_measurments() < min_num_antennas:
                continue
            
            fitter.prep_for_fitting()
            
            X = brute(fitter.RMS, ranges=[[0,np.pi/2],[0,np.pi*2]], Ns=20)
            
            if X[0] >= np.pi/2:
                X[0] = (np.pi/2)*0.99
            if X[1] >= np.pi*2:
                X[1] = (np.pi*2)*0.99
                
            if X[0] < 0:
                X[0] = 0.00000001
            if X[1] < 0:
                X[1] = 0.00000001
            
            ret = least_squares( fitter.time_fits, X, bounds=[[0,0],[np.pi/2,2*np.pi]], ftol=3e-16, xtol=3e-16, gtol=3e-16, x_scale='jac' )
            
            out_RMS[N] = fitter.RMS(ret.x, 3)
            out_zenith[N] = ret.x[0] 
            out_azimuth[N] = ret.x[1] 
            N += 1
            
            i += 1
            
    return out_RMS[:N], out_zenith[:N], out_azimuth[:N]
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
コード例 #10
0
def run_fitter(timeID,
               output_folder,
               pulse_input_folders,
               guess_timings,
               souces_to_fit,
               guess_source_locations,
               source_polarizations,
               source_stations_to_exclude,
               source_antennas_to_exclude,
               bad_ants,
               ref_station="CS002",
               min_ant_amplitude=10,
               num_itters=1000,
               error_deviation=0.5E-9,
               antenna_error=0.5E-9):

    ##### holdovers. These globals need to be fixed, so not global....
    global station_locations, station_to_antenna_index_list, stations_with_fits, station_to_antenna_index_dict
    global referance_station, station_order, sorted_antenna_names, min_antenna_amplitude, ant_locs, bad_antennas
    global current_delays_guess, processed_data_folder

    referance_station = ref_station
    min_antenna_amplitude = min_ant_amplitude
    bad_antennas = bad_ants

    if referance_station in guess_timings:
        ref_T = guess_timings[referance_station]
        guess_timings = {
            station: T - ref_T
            for station, T in guess_timings.items()
            if station != referance_station
        }

    processed_data_folder = processed_data_dir(timeID)

    data_dir = processed_data_folder + "/" + output_folder
    if not isdir(data_dir):
        mkdir(data_dir)

    logging_folder = data_dir + '/logs_and_plots'
    if not isdir(logging_folder):
        mkdir(logging_folder)

    #Setup logger and open initial data set
    log = logger()
    log.set(
        logging_folder +
        "/log_out.txt")  ## TODo: save all output to a specific output folder
    log.take_stderr()
    log.take_stdout()

    print("timeID:", timeID)
    print("date and time run:", time.strftime("%c"))
    print("input folders:", pulse_input_folders)
    print("source IDs to fit:", souces_to_fit)
    print("guess locations:", guess_source_locations)
    print("polarization to use:", source_polarizations)
    print("source stations to exclude:", source_stations_to_exclude)
    print("source antennas to exclude:", source_antennas_to_exclude)
    print("bad antennas:", bad_ants)
    print("referance station:", ref_station)
    print("guess delays:", guess_timings)
    print('pulse error:', error_deviation)
    print('antenna error:', antenna_error)
    print()

    #### open data and data processing stuff ####
    print("loading data")
    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {
        sname: MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
        for sname, fpaths in raw_fpaths.items()
        if sname in chain(guess_timings.keys(), [referance_station])
    }

    #### sort antennas and stations ####
    station_order = list(
        guess_timings.keys())  ## note this doesn't include reference station
    sorted_antenna_names = []
    station_to_antenna_index_dict = {}
    ant_loc_dict = {}

    for sname in station_order + [referance_station]:
        first_index = len(sorted_antenna_names)

        stat_data = raw_data_files[sname]
        even_ant_names = stat_data.get_antenna_names()[::2]
        even_ant_locs = stat_data.get_LOFAR_centered_positions()[::2]

        sorted_antenna_names += even_ant_names

        for ant_name, ant_loc in zip(even_ant_names, even_ant_locs):
            ant_loc_dict[ant_name] = ant_loc

        station_to_antenna_index_dict[sname] = (first_index,
                                                len(sorted_antenna_names))

    ant_locs = np.zeros((len(sorted_antenna_names), 3))
    for i, ant_name in enumerate(sorted_antenna_names):
        ant_locs[i] = ant_loc_dict[ant_name]

    station_locations = {
        sname: ant_locs[station_to_antenna_index_dict[sname][0]]
        for sname in station_order + [referance_station]
    }
    station_to_antenna_index_list = [
        station_to_antenna_index_dict[sname]
        for sname in station_order + [referance_station]
    ]

    #### sort the delays guess, and account for station locations ####
    current_delays_guess = np.array(
        [guess_timings[sname] for sname in station_order])
    #    original_delays = np.array( current_delays_guess )

    #### open info from part 1 ####

    input_manager = Part1_input_manager(pulse_input_folders)

    #### first we fit the known sources ####
    current_sources = []
    #    next_source = 0
    for knownID in souces_to_fit:

        source_ID, input_name = input_manager.known_source(knownID)

        print("prep fitting:", source_ID)

        location = guess_source_locations[source_ID]

        ## make source
        source_to_add = source_object(source_ID, input_name, location,
                                      source_stations_to_exclude[source_ID],
                                      source_antennas_to_exclude[source_ID],
                                      num_itters)
        current_sources.append(source_to_add)

        polarity = source_polarizations[source_ID]

        source_to_add.prep_for_fitting(polarity, guess_timings)

    fitter = stochastic_fitter_dt(current_sources)

    all_delays = np.empty((num_itters, fitter.num_delays), dtype=np.double)
    all_RMSs = np.empty(num_itters, dtype=np.double)
    for i in range(num_itters):
        all_delays[i, :], all_RMSs[i] = fitter.rerun(error_deviation,
                                                     antenna_error)

        fitter.employ_result(current_sources)
        print('run', i, 'RMS:', all_RMSs[i])

    print()
    print()
    print("station timing errors:")
    for i, sname in zip(range(fitter.num_delays), station_order):
        print(sname, ":", np.std(all_delays[:, i]))

    print()
    print()

    ### get average X, Y, Z for each itteraion
    ave_X = np.zeros(num_itters)
    ave_Y = np.zeros(num_itters)
    ave_Z = np.zeros(num_itters)

    for source in current_sources:
        ave_X += source.solutions[:, 0]
        ave_Y += source.solutions[:, 1]
        ave_Z += source.solutions[:, 2]

    ave_X /= len(current_sources)
    ave_Y /= len(current_sources)
    ave_Z /= len(current_sources)

    print("absolute location errors:")
    print("X", np.std(ave_X), "Y", np.std(ave_Y), "Z", np.std(ave_Z))

    print()
    print()
    print("relative location errors")

    for source in current_sources:
        source.solutions[:, 0] -= ave_X
        source.solutions[:, 1] -= ave_Y
        source.solutions[:, 2] -= ave_Z

        print("source", source.ID)
        print("  ", np.std(source.solutions[:, 0]),
              np.std(source.solutions[:, 1]), np.std(source.solutions[:, 2]))

    print()
    print()

    print("average RMS", np.average(all_RMSs), "std of RMS", np.std(all_RMSs))
コード例 #11
0
    def run(self, output_dir, dir_is_organized=True):
        
        processed_data_folder = processed_data_dir( self.timeID )
        
        if dir_is_organized:
            output_dir = processed_data_folder +'/' + output_dir
        
        polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + self.pol_flips_fname )
        bad_antennas = read_bad_antennas( processed_data_folder + '/' + self.bad_antennas_fname )
        additional_antenna_delays = read_antenna_delays(  processed_data_folder + '/' + self.additional_antenna_delays_fname )
        station_timing_offsets = read_station_delays( processed_data_folder+'/'+ self.station_delays_fname )
        
        raw_fpaths = filePaths_by_stationName( self.timeID )

        self.station_data_files = []
        ref_station_i = None
        referance_station_set = None
        station_i = 0
        self.posix_timestamp = None
        
        for station, fpaths  in raw_fpaths.items():
            if (station in station_timing_offsets) and (station not in self.stations_to_exclude ):
                print('opening', station)
                
                raw_data_file = MultiFile_Dal1(fpaths, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays, pol_flips_are_bad=self.pol_flips_are_bad)
                self.station_data_files.append( raw_data_file )
                
                raw_data_file.set_station_delay( station_timing_offsets[station] )
                raw_data_file.find_and_set_polarization_delay()
                
                if self.posix_timestamp is None:
                    self.posix_timestamp = raw_data_file.get_timestamp()
                elif self.posix_timestamp != raw_data_file.get_timestamp():
                    print("ERROR: POSIX timestamps are different")
                    quit()
                
                if self.referance_station is None:
                    if (referance_station_set is None) or ( int(station[2:]) < int(referance_station_set[2:]) ): ## if no referance station, use one with smallist number
                        ref_station_i = station_i
                        referance_station_set = station
                elif self.referance_station == station:
                    ref_station_i = station_i
                    referance_station_set = station
                        
                station_i += 1
                
        ## set reference station first
        tmp = self.station_data_files[0]
        self.station_data_files[0] = self.station_data_files[ ref_station_i ]
        self.station_data_files[ ref_station_i ] = tmp
        self.referance_station = referance_station_set
        
        ## organize antenana info
        self.station_antenna_indeces = [] ## 2D list. First index is station_i, second is a local antenna index, value is station antenna index
        self.station_antenna_RMS = [] ## same as above, but value is RMS
        self.num_total_antennas = 0
        current_ant_i = 0
        for station_i,stationFile in enumerate(self.station_data_files):
            ant_names = stationFile.get_antenna_names()
            num_evenAntennas = int( len(ant_names)/2 )
            
            ## get RMS planewave fits
            if self.antenna_RMS_info is not None:
                
                antenna_RMS_dict = read_planewave_fits(processed_data_folder+'/'+self.antenna_RMS_info, stationFile.get_station_name() )
            else:
                antenna_RMS_dict = {}
                
                
            antenna_RMS_array = np.array([ antenna_RMS_dict[ant_name] if ant_name in antenna_RMS_dict else self.default_expected_RMS  for ant_name in ant_names if antName_is_even(ant_name)  ])
            antenna_indeces = np.arange(num_evenAntennas)*2
            
            ## remove bad antennas and sort
            ant_is_good = np.isfinite( antenna_RMS_array )
            antenna_RMS_array = antenna_RMS_array[ ant_is_good ]
            antenna_indeces = antenna_indeces[ ant_is_good ]
            
            sorter = np.argsort( antenna_RMS_array )
            antenna_RMS_array = antenna_RMS_array[ sorter ]
            antenna_indeces = antenna_indeces[ sorter ]
            
            ## select only the best!
            antenna_indeces = antenna_indeces[:self.max_antennas_per_station]
            antenna_RMS_array = antenna_RMS_array[:self.max_antennas_per_station]
                
            ## fill info
            self.station_antenna_indeces.append( antenna_indeces )
            self.station_antenna_RMS.append( antenna_RMS_array )
            
            self.num_total_antennas += len(antenna_indeces)
            
            current_ant_i += len(antenna_indeces)
        
        ### output to header!
        if not isdir(output_dir):
            mkdir(output_dir)
            
        logging_folder = output_dir + '/logs_and_plots'
        if not isdir(logging_folder):
            mkdir(logging_folder)
            
        header_outfile = h5py.File(output_dir + "/header.h5", "w")
        header_outfile.attrs["timeID"] = self.timeID
        header_outfile.attrs["initial_datapoint"] = self.initial_datapoint
        header_outfile.attrs["max_antennas_per_station"] = self.max_antennas_per_station
        header_outfile.attrs["referance_station"] = self.station_data_files[0].get_station_name()
        header_outfile.attrs["station_delays_fname"] = self.station_delays_fname
        header_outfile.attrs["additional_antenna_delays_fname"] = self.additional_antenna_delays_fname
        header_outfile.attrs["bad_antennas_fname"] = self.bad_antennas_fname
        header_outfile.attrs["pol_flips_fname"] = self.pol_flips_fname
        header_outfile.attrs["blocksize"] = self.blocksize
        header_outfile.attrs["remove_saturation"] = self.remove_saturation
        header_outfile.attrs["remove_RFI"] = self.remove_RFI
        header_outfile.attrs["positive_saturation"] = self.positive_saturation
        header_outfile.attrs["negative_saturation"] = self.negative_saturation
        header_outfile.attrs["saturation_removal_length"] = self.saturation_removal_length
        header_outfile.attrs["saturation_half_hann_length"] = self.saturation_half_hann_length
        header_outfile.attrs["hann_window_fraction"] = self.hann_window_fraction
        header_outfile.attrs["num_zeros_dataLoss_Threshold"] = self.num_zeros_dataLoss_Threshold
        header_outfile.attrs["min_amplitude"] = self.min_amplitude
        header_outfile.attrs["upsample_factor"] = self.upsample_factor
        header_outfile.attrs["max_events_perBlock"] = self.max_events_perBlock
        header_outfile.attrs["min_pulse_length_samples"] = self.min_pulse_length_samples
        header_outfile.attrs["erasure_length"] = self.erasure_length
        header_outfile.attrs["guess_distance"] = self.guess_distance
        header_outfile.attrs["kalman_devations_toSearch"] = self.kalman_devations_toSearch
        header_outfile.attrs["pol_flips_are_bad"] = self.pol_flips_are_bad
        header_outfile.attrs["antenna_RMS_info"] = self.antenna_RMS_info
        header_outfile.attrs["default_expected_RMS"] = self.default_expected_RMS
        header_outfile.attrs["max_planewave_RMS"] = self.max_planewave_RMS
        header_outfile.attrs["stop_chi_squared"] = self.stop_chi_squared
        header_outfile.attrs["max_minimize_itters"] = self.max_minimize_itters
        header_outfile.attrs["minimize_ftol"] = self.minimize_ftol
        header_outfile.attrs["minimize_xtol"] = self.minimize_xtol
        header_outfile.attrs["minimize_gtol"] = self.minimize_gtol
        
        header_outfile.attrs["refStat_delay"] = station_timing_offsets[  self.station_data_files[0].get_station_name()  ]
        header_outfile.attrs["refStat_timestamp"] = self.posix_timestamp#self.station_data_files[0].get_timestamp()
        header_outfile.attrs["refStat_sampleNumber"] = self.station_data_files[0].get_nominal_sample_number()
        
        header_outfile.attrs["stations_to_exclude"] = np.array(self.stations_to_exclude, dtype='S')
        
        header_outfile.attrs["polarization_flips"] = np.array(polarization_flips, dtype='S')
        
        header_outfile.attrs["num_stations"] = len(self.station_data_files)
        header_outfile.attrs["num_antennas"] = self.num_total_antennas
        
        for stat_i, (stat_file, antenna_indeces, antenna_RMS) in enumerate(zip(self.station_data_files, self.station_antenna_indeces, self.station_antenna_RMS)):
            
            station_group = header_outfile.create_group( str(stat_i) )
            station_group.attrs["sname"] = stat_file.get_station_name()
            station_group.attrs["num_antennas"] = len( antenna_indeces )
            
            locations = stat_file.get_LOFAR_centered_positions()
            total_delays = stat_file.get_total_delays()
            ant_names = stat_file.get_antenna_names()
            
            for ant_i, (stat_index, RMS) in enumerate(zip(antenna_indeces, antenna_RMS)):
                
                antenna_group = station_group.create_group( str(ant_i) )
                antenna_group.attrs['antenna_name'] = ant_names[stat_index]
                antenna_group.attrs['location'] = locations[stat_index]
                antenna_group.attrs['delay'] = total_delays[stat_index]
                antenna_group.attrs['planewave_RMS'] = RMS
コード例 #12
0
utilities.default_processed_data_loc = "/home/student4/Marten/processed_files"
processed_data_folder = processed_data_dir(timeID)

station_delay_file = "station_delays.txt"
polarization_flips = "polarization_flips.txt"
bad_antennas = "bad_antennas.txt"
additional_antenna_delays = "ant_delays.txt"

polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' + polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' + additional_antenna_delays)
station_timing_offsets = read_station_delays(processed_data_folder + '/' + station_delay_file)

raw_fpaths = filePaths_by_stationName(timeID)

TBB_data = {sname : MultiFile_Dal1(fpath, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas,
additional_ant_delays=additional_antenna_delays, only_complete_pairs=True) for sname, fpath in raw_fpaths.items() if sname in station_timing_offsets}

def model(p, r):
	ϕ = np.arctan2(r[:,1], r[:,0])
	ϕ_hat = np.array([-np.sin(ϕ), np.cos(ϕ), np.zeros(ϕ.size)]).T #unit azimuth vector
	
	r_hat = r/np.linalg.norm(r, axis=1)[:, np.newaxis] #radial unit vector to source
	
	a = np.array([np.sin(p[1])*np.cos(p[0]), np.sin(p[1])*np.sin(p[0]), np.cos(p[1])])
	a_perp = a - np.einsum('j,ij,ik->ik', a, r_hat, r_hat)
	a_perp_hat = a_perp/np.linalg.norm(a_perp, axis=1)[:, np.newaxis]
	
	τ = np.arccos(np.einsum('ij,ij->i', a_perp_hat, ϕ_hat))
	return τ

def residuals(p, data):
コード例 #13
0
    def open_files(self, log_func=do_nothing, force_metadata_ant_pos=True):
        processed_data_loc = processed_data_dir(self.timeID)

        #### open callibration data files ####
        ### TODO: fix these so they are accounted for in a more standard maner
        self.station_timing_offsets = read_station_delays(
            processed_data_loc + '/' + self.station_delays_fname)
        self.additional_ant_delays = read_antenna_delays(
            processed_data_loc + '/' + self.additional_antenna_delays_fname)
        self.bad_antennas = read_bad_antennas(processed_data_loc + '/' +
                                              self.bad_antennas_fname)
        self.pol_flips = read_antenna_pol_flips(processed_data_loc + '/' +
                                                self.pol_flips_fname)

        #### open data files, and find RFI ####
        raw_fpaths = filePaths_by_stationName(self.timeID)
        self.station_names = []
        self.input_files = []
        self.RFI_filters = []
        CS002_index = None
        self.station_to_antenna_indeces_dict = {}
        for station, fpaths in raw_fpaths.items():
            if (station in self.station_timing_offsets) and (
                    station not in self.stations_to_exclude) and (
                        self.use_core_stations_S1 or self.use_core_stations_S2
                        or (not station[:2] == 'CS') or station == "CS002"):
                #            if (station not in self.stations_to_exclude) and ( self.use_core_stations_S1 or self.use_core_stations_S2 or (not station[:2]=='CS') or station=="CS002" ):
                log_func("opening", station)
                self.station_names.append(station)
                self.station_to_antenna_indeces_dict[station] = []

                if station == 'CS002':
                    CS002_index = len(self.station_names) - 1

                new_file = MultiFile_Dal1(
                    fpaths, force_metadata_ant_pos=force_metadata_ant_pos)
                new_file.set_polarization_flips(self.pol_flips)
                self.input_files.append(new_file)

                RFI_result = None
                if self.do_RFI_filtering and self.use_saved_RFI_info:
                    self.RFI_filters.append(
                        window_and_filter(timeID=self.timeID, sname=station))

                elif self.do_RFI_filtering:
                    RFI_result = FindRFI(new_file,
                                         self.block_size,
                                         self.initial_RFI_block,
                                         self.RFI_num_blocks,
                                         self.RFI_max_blocks,
                                         verbose=False,
                                         figure_location=None)
                    self.RFI_filters.append(
                        window_and_filter(find_RFI=RFI_result))

                else:  ## only basic filtering
                    self.RFI_filters.append(
                        window_and_filter(blocksize=self.block_size))

        #### find antenna pairs ####
        boundingBox_center = np.average(self.bounding_box, axis=-1)
        self.antennas_to_use = pairData_NumAntPerStat(
            self.input_files, self.num_antennas_per_station, self.bad_antennas)
        #        self.antennas_to_use = pairData_NumAntPerStat_PolO(self.input_files, self.num_antennas_per_station, self.bad_antennas )
        #        self.antennas_to_use = pairData_NumAntPerStat_DualPol(self.input_files, self.num_antennas_per_station, self.bad_antennas )
        self.num_antennas = len(self.antennas_to_use)

        #### get antenna locations and delays ####
        self.antenna_locations = np.zeros((self.num_antennas, 3),
                                          dtype=np.double)
        self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double)
        self.is_not_core = np.zeros(self.num_antennas, dtype=np.bool)

        #### TODO: Do we need this "CSOO2 correction??? will removing it fix our time base?? ####
        #        clock_corrections = getClockCorrections()
        #        CS002_correction = -clock_corrections["CS002"] - self.input_files[CS002_index].get_nominal_sample_number()*5.0E-9 ## wierd sign. But is just to correct for previous definiitions
        CS002_correction = self.station_timing_offsets[
            'CS002'] - self.input_files[CS002_index].get_nominal_sample_number(
            ) * 5.0E-9  ## wierd sign. But is just to correct for previous definiitions

        for ant_i, (station_i,
                    station_ant_i) in enumerate(self.antennas_to_use):
            self.station_to_antenna_indeces_dict[
                self.station_names[station_i]].append(
                    ant_i
                )  ### TODO: this should probably be a result of pairData_NumAntPerStat

            data_file = self.input_files[station_i]
            station = self.station_names[station_i]
            self.is_not_core[ant_i] = (not station[:2]
                                       == 'CS') or station == "CS002"

            ant_name = data_file.get_antenna_names()[station_ant_i]

            self.antenna_locations[
                ant_i] = data_file.get_LOFAR_centered_positions(
                )[station_ant_i]
            self.antenna_delays[
                ant_i] = data_file.get_timing_callibration_delays(
                )[station_ant_i]

            ## account for station timing offsets
            #            self.antenna_delays[ant_i]  += self.station_timing_offsets[station] +  (-clock_corrections[station] - data_file.get_nominal_sample_number()*5.0E-9) - CS002_correction
            self.antenna_delays[ant_i] += self.station_timing_offsets[
                station] + (-data_file.get_nominal_sample_number() *
                            5.0E-9) - CS002_correction

            ## add additional timing delays
            ##note this only works for even antennas!
            if ant_name in self.additional_ant_delays:
                self.antenna_delays[ant_i] += self.additional_ant_delays[
                    ant_name][0]

#        #### find prefered station ####
        if self.prefered_station is None:
            prefered_stat_shortestWindowTime = np.inf
            prefered_station_input_file = None
            for station, antenna_indeces in self.station_to_antenna_indeces_dict.items(
            ):
                ant_i = antenna_indeces[0]  ## just use first antenna for test

                if not (self.use_core_stations_S1 or self.is_not_core[ant_i]):
                    continue

                longest_window_time = 0
                for ant_j in range(self.num_antennas):
                    if not (self.use_core_stations_S1
                            or self.is_not_core[ant_j]):
                        continue

                    window_time, throw, throw = find_max_duration(
                        self.antenna_locations[ant_i],
                        self.antenna_locations[ant_j],
                        self.bounding_box,
                        center=boundingBox_center)
                    if window_time > longest_window_time:
                        longest_window_time = window_time

                if longest_window_time < prefered_stat_shortestWindowTime:
                    prefered_stat_shortestWindowTime = longest_window_time
                    self.prefered_station = station
                    prefered_station_input_file = self.input_files[
                        self.antennas_to_use[ant_i][0]]
        else:  ### TODO: throw error is prefered station is in core, but core not included in stage 1
            ant_i = self.station_to_antenna_indeces_dict[
                self.prefered_station][0]

            prefered_stat_shortestWindowTime = 0
            for ant_j in range(self.num_antennas):
                if not (self.use_core_stations_S1 or self.is_not_core[ant_j]):
                    continue

                window_time, throw, throw = find_max_duration(
                    self.antenna_locations[ant_i],
                    self.antenna_locations[ant_j],
                    self.bounding_box,
                    center=boundingBox_center)
                if window_time > prefered_stat_shortestWindowTime:
                    prefered_stat_shortestWindowTime = window_time

            prefered_station_input_file = self.input_files[
                self.antennas_to_use[ant_i][0]]

        log_func("prefered station:", self.prefered_station)
        log_func("    max. half window time:",
                 prefered_stat_shortestWindowTime)
        log_func()
        log_func()

        self.prefered_station_timing_offset = self.station_timing_offsets[
            self.
            prefered_station] - prefered_station_input_file.get_nominal_sample_number(
            ) * 5.0E-9
        self.prefered_station_antenna_timing_offsets = prefered_station_input_file.get_timing_callibration_delays(
        )

        self.startBlock_exclusion = int(
            0.1 * self.block_size)  ### TODO: save these to header.
        self.endBlock_exclusion = int(
            prefered_stat_shortestWindowTime / 5.0E-9) + 1 + int(
                0.1 * self.block_size)  ## last bit accounts for hann-window
        self.active_block_length = self.block_size - self.startBlock_exclusion - self.endBlock_exclusion  ##the length of block used for looking at data

        ##### find window offsets and lengths ####
        self.antenna_data_offsets = np.zeros(self.num_antennas, dtype=np.long)
        self.half_antenna_data_length = np.zeros(self.num_antennas,
                                                 dtype=np.long)

        for ant_i, (station_i,
                    station_ant_i) in enumerate(self.antennas_to_use):

            ## now we account for distance to the source
            travel_time = np.linalg.norm(self.antenna_locations[ant_i] -
                                         boundingBox_center) / v_air
            self.antenna_data_offsets[ant_i] = int(travel_time / 5.0E-9)

            ### find max duration to any of the prefered antennas
            max_duration = 0.0
            for prefered_ant_i in self.station_to_antenna_indeces_dict[
                    self.prefered_station]:
                if prefered_ant_i == ant_i:
                    continue

                duration, throw, throw = find_max_duration(
                    self.antenna_locations[prefered_ant_i],
                    self.antenna_locations[ant_i], self.bounding_box,
                    boundingBox_center)

                if duration > max_duration:
                    max_duration = duration

            self.half_antenna_data_length[ant_i] = int(
                self.pulse_length / 2) + int(duration / 5.0E-9)

            self.antenna_data_offsets[ant_i] -= self.half_antenna_data_length[
                ant_i]

            #### now adjust the data offsets and antenna delays so they are consistent

            ## first we amount to adjust the offset by time delays mod the sampling time
            offset_adjust = int(
                self.antenna_delays[ant_i] / 5.0E-9
            )  ##this needs to be added to offsets and subtracted from delays

            ## then we can adjust the delays accounting for the data offset
            self.antenna_delays[
                ant_i] -= self.antenna_data_offsets[ant_i] * 5.0E-9

            ##now we finally account for large time delays
            self.antenna_data_offsets[ant_i] += offset_adjust
            self.antenna_delays[ant_i] -= offset_adjust * 5.0E-9

        if (not self.use_core_stations_S1) or (not self.use_core_stations_S2):
            core_filtered_ant_locs = np.array(
                self.antenna_locations[self.is_not_core])
            core_filtered_ant_delays = np.array(
                self.antenna_delays[self.is_not_core])

        #### allocate some memory ####
        self.data_block = np.empty((self.num_antennas, self.block_size),
                                   dtype=np.complex)
        self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double)

        #### initialize stage 1 ####
        if self.use_core_stations_S1:
            self.trace_length_stage1 = 2 * np.max(
                self.half_antenna_data_length)
            S1_ant_locs = self.antenna_locations
            S1_ant_delays = self.antenna_delays
        else:
            self.trace_length_stage1 = 2 * np.max(
                self.half_antenna_data_length[self.is_not_core])
            S1_ant_locs = core_filtered_ant_locs
            S1_ant_delays = core_filtered_ant_delays
        self.trace_length_stage1 = 2**(int(np.log2(self.trace_length_stage1)) +
                                       1)
        self.stage_1_imager = II_tools.image_data_stage1(
            S1_ant_locs, S1_ant_delays, self.trace_length_stage1,
            self.upsample_factor)

        #### initialize stage 2 ####
        if self.use_core_stations_S2:
            S2_ant_locs = self.antenna_locations
            S2_ant_delays = self.antenna_delays
        else:
            S2_ant_locs = core_filtered_ant_locs
            S2_ant_delays = core_filtered_ant_delays
        self.trace_length_stage2 = 2**(int(np.log2(self.pulse_length)) + 1)
        self.stage_2_window = half_hann_window(self.pulse_length,
                                               self.hann_window_fraction)
        self.stage_2_imager = II_tools.image_data_stage2_absBefore(
            S2_ant_locs, S2_ant_delays, self.trace_length_stage2,
            self.upsample_factor)

        self.erasure_window = 1.0 - self.stage_2_window
コード例 #14
0
def save_EventByLoc(timeID, XYZT, station_timing_delays, pulse_index, output_folder, pulse_width=50, min_ant_amp=5, upsample_factor=0,
                    polarization_flips="polarization_flips.txt", bad_antennas="bad_antennas.txt", additional_antenna_delays = "ant_delays.txt"):
    
    processed_data_folder = processed_data_dir(timeID)
    
    polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
    bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
    additional_antenna_delays = read_antenna_delays(  processed_data_folder + '/' + additional_antenna_delays )
    
    data_dir = processed_data_folder + "/" + output_folder
    if not isdir(data_dir):
        mkdir(data_dir)
        
    logging_folder = data_dir + '/logs_and_plots'
    if not isdir(logging_folder):
        mkdir(logging_folder)

    #Setup logger and open initial data set
    log = logger()
    log.set(logging_folder + "/log_out_"+str(pulse_index)+"_.txt") ## TODo: save all output to a specific output folder
    log.take_stderr()
    log.take_stdout()
    
    print("saving traces to file", pulse_index)
    print("location:", XYZT)
    print("upsample_factor:", upsample_factor)
    
    print()
    print()
    print("station timing offsets")
    print(station_timing_delays)
    
    print()
    print()
    print("pol flips")
    print(polarization_flips)
    
    print()
    print()
    print("bad antennas")
    print(bad_antennas)
    
    print()
    print()
    print("additional antenna delays")
    print(additional_antenna_delays)
    
    
    
    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays,
                                           station_delay=station_timing_delays[sname]) for sname,fpaths in raw_fpaths.items() if sname in station_timing_delays}
    
    data_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_timing_delays}
    
    trace_locator = getTrace_fromLoc( raw_data_files, data_filters )
    
    print()
    print()
    print("data opened")
    
    out_fname = data_dir + "/potSource_"+str(pulse_index)+".h5"
    out_file = h5py.File(out_fname, "w")
    for sname in station_timing_delays.keys():
        TBB_file = raw_data_files[ sname ]
        TBB_file.find_and_set_polarization_delay()
        antenna_names = TBB_file.get_antenna_names()
        
        
        h5_statGroup = out_file.create_group( sname )
        print()
        print(sname)
        
        for even_ant_i in range(0,len(antenna_names),2):
            even_ant_name = antenna_names[ even_ant_i ]
            odd_ant_name = antenna_names[ even_ant_i+1 ]
            
            starting_index, PolE_offset, throw, PolE_trace = trace_locator.get_trace_fromLoc(XYZT, even_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
            throw, PolO_offset, throw, PolO_trace = trace_locator.get_trace_fromIndex(starting_index, odd_ant_name, pulse_width, do_remove_RFI=True, do_remove_saturation=True)
            
            PolE_HE = np.abs(PolE_trace)
            PolO_HE = np.abs(PolO_trace)
            
            
            h5_Ant_dataset = h5_statGroup.create_dataset(even_ant_name, (4, pulse_width ), dtype=np.double)
            
            h5_Ant_dataset[0] = np.real(PolE_trace)
            h5_Ant_dataset[1] = PolE_HE
            h5_Ant_dataset[2] = np.real(PolO_trace)
            h5_Ant_dataset[3] = PolO_HE
            
            ### note that peak time should NOT account for station timing offsets!
            
            h5_Ant_dataset.attrs['starting_index'] = starting_index
            h5_Ant_dataset.attrs['PolE_timeOffset'] = -(PolE_offset - station_timing_delays[sname])
            h5_Ant_dataset.attrs['PolO_timeOffset'] = -(PolO_offset - station_timing_delays[sname])
            h5_Ant_dataset.attrs['PolE_timeOffset_CS'] = (PolE_offset - station_timing_delays[sname])
            h5_Ant_dataset.attrs['PolO_timeOffset_CS'] = (PolO_offset - station_timing_delays[sname])
            
            
            
            
            sample_time = 5.0E-9
            if upsample_factor > 1:
                PolE_HE = resample(PolE_HE, len(PolE_HE)*upsample_factor )
                PolO_HE = resample(PolO_HE, len(PolO_HE)*upsample_factor )
                
                sample_time /= upsample_factor
            
            
            if np.max(PolE_HE)> min_ant_amp:
                
                
                PolE_peak_finder = parabolic_fit( PolE_HE )
                h5_Ant_dataset.attrs['PolE_peakTime'] =  starting_index*5.0E-9 - (PolE_offset-station_timing_delays[sname]) + PolE_peak_finder.peak_index*sample_time
            else:
                h5_Ant_dataset.attrs['PolE_peakTime'] = np.nan
            
            if np.max(PolO_HE)> min_ant_amp:
                
                
                PolO_peak_finder = parabolic_fit( PolO_HE )
                h5_Ant_dataset.attrs['PolO_peakTime'] =  starting_index*5.0E-9 - (PolO_offset-station_timing_delays[sname]) + PolO_peak_finder.peak_index*sample_time
            else:
                h5_Ant_dataset.attrs['PolO_peakTime'] =  np.nan
            
            print("  ", even_ant_name, h5_Ant_dataset.attrs['PolE_peakTime'], h5_Ant_dataset.attrs['PolO_peakTime'])
                
            
    out_file.close()
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
    
def run_fitter(timeID, output_folder, pulse_input_folders, guess_timings, souces_to_fit, guess_source_locations,
               source_polarizations, source_stations_to_exclude, source_antennas_to_exclude, bad_ants,
               X_deviations, Y_deviations, Z_deviations,
               ref_station="CS002", min_ant_amplitude=10, max_stoch_loop_itters = 2000, min_itters_till_convergence = 100,
               initial_jitter_width = 100000E-9, final_jitter_width = 1E-9, cooldown_fraction = 10.0, strong_cooldown_fraction = 100.0,
               fitter = "dt"):
    
    ##### holdovers. These globals need to be fixed, so not global....
    global station_locations, station_to_antenna_index_list, stations_with_fits, station_to_antenna_index_dict
    global referance_station, station_order, sorted_antenna_names, min_antenna_amplitude, ant_locs, bad_antennas
    global max_itters_per_loop, itters_till_convergence, min_jitter_width, max_jitter_width, cooldown, strong_cooldown
    global current_delays_guess, processed_data_folder
    
    referance_station = ref_station
    min_antenna_amplitude = min_ant_amplitude
    bad_antennas = bad_ants
    max_itters_per_loop = max_stoch_loop_itters
    itters_till_convergence = min_itters_till_convergence
    max_jitter_width = initial_jitter_width
    min_jitter_width = final_jitter_width
    cooldown = cooldown_fraction
    strong_cooldown = strong_cooldown_fraction
    
    if referance_station in guess_timings:
        ref_T = guess_timings[referance_station]
        guess_timings = {station:T-ref_T for station,T in guess_timings.items() if station != referance_station}
        
    processed_data_folder = processed_data_dir(timeID)
    
    data_dir = processed_data_folder + "/" + output_folder
    if not isdir(data_dir):
        mkdir(data_dir)
        
    logging_folder = data_dir + '/logs_and_plots'
    if not isdir(logging_folder):
        mkdir(logging_folder)



    #Setup logger and open initial data set
    log = logger()
    log.set(logging_folder + "/log_out.txt") ## TODo: save all output to a specific output folder
    log.take_stderr()
    log.take_stdout()
    
    
    print("timeID:", timeID)
    print("date and time run:", time.strftime("%c") )
    print("input folders:", pulse_input_folders)
    print("source IDs to fit:", souces_to_fit)
    print("guess locations:", guess_source_locations)
    print("polarization to use:", source_polarizations)
    print("source stations to exclude:", source_stations_to_exclude)
    print("source antennas to exclude:", source_antennas_to_exclude)
    print("bad antennas:", bad_ants)
    print("referance station:", ref_station)
    print("fitter type:", fitter)
    print("guess delays:", guess_timings)
    print()
    print()
    
        #### open data and data processing stuff ####
    print("loading data")
    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True) for sname,fpaths in raw_fpaths.items() if sname in chain(guess_timings.keys(), [referance_station]) }
    
    
    
    
    #### sort antennas and stations ####
    station_order = list(guess_timings.keys())## note this doesn't include reference station
    sorted_antenna_names = []
    station_to_antenna_index_dict = {}
    ant_loc_dict = {}
    
    for sname in station_order + [referance_station]:
        first_index = len(sorted_antenna_names)
        
        stat_data = raw_data_files[sname]
        even_ant_names = stat_data.get_antenna_names()[::2]
        even_ant_locs = stat_data.get_LOFAR_centered_positions()[::2]
        
        sorted_antenna_names += even_ant_names
        
        for ant_name, ant_loc in zip(even_ant_names,even_ant_locs):
            ant_loc_dict[ant_name] = ant_loc
                
        station_to_antenna_index_dict[sname] = (first_index, len(sorted_antenna_names))
    
    ant_locs = np.zeros( (len(sorted_antenna_names), 3))
    for i, ant_name in enumerate(sorted_antenna_names):
        ant_locs[i] = ant_loc_dict[ant_name]
    
    station_locations = {sname:ant_locs[station_to_antenna_index_dict[sname][0]] for sname in station_order + [referance_station]}
    station_to_antenna_index_list = [station_to_antenna_index_dict[sname] for sname in station_order + [referance_station]]
    
    
    #### sort the delays guess, and account for station locations ####
    current_delays_guess = np.array([guess_timings[sname] for sname in station_order])
    original_delays = np.array( current_delays_guess )        
    
    
    
    
    #### open info from part 1 ####

    input_manager = Part1_input_manager( pulse_input_folders )



    #### first we fit the known sources ####
    current_sources = []
#    next_source = 0
    for knownID in souces_to_fit:
        
        source_ID, input_name = input_manager.known_source( knownID )
        
        print("prep fitting:", source_ID)
            
            
        location = guess_source_locations[source_ID]
        
        ## make source
        source_to_add = source_object(source_ID, input_name, location, source_stations_to_exclude[source_ID], source_antennas_to_exclude[source_ID] )
        current_sources.append( source_to_add )


        polarity = source_polarizations[source_ID]

            
        source_to_add.prep_for_fitting(polarity)
        
    print()
    print("fitting known sources")
    if fitter!='dt':
        print("only dt")
        quit()
        
    first_source = current_sources[0]
    original_XYZT = np.array( first_source.guess_XYZT )
    
    if X_deviations is not None:
        RMS_values = np.empty( len(X_deviations) )
        
        first_source.guess_XYZT[:] = original_XYZT
        for i in range(len(X_deviations)):
        
            first_source.guess_XYZT[0] = original_XYZT[0] + X_deviations[i]
            fitter = stochastic_fitter_dt(current_sources)
            RMS_values[i] = fitter.RMS
            
            print('X', X_deviations[i], fitter.RMS)
            
        plt.plot(X_deviations, RMS_values)
        plt.show()
        
    
    if Y_deviations is not None:
        RMS_values = np.empty( len(Y_deviations) )
        
        first_source.guess_XYZT[:] = original_XYZT
        for i in range(len(Y_deviations)):
        
            first_source.guess_XYZT[1] = original_XYZT[1] + Y_deviations[i]
            fitter = stochastic_fitter_dt(current_sources)
            RMS_values[i] = fitter.RMS
            
            print('Y', Y_deviations[i], fitter.RMS)
            
        plt.plot(Y_deviations, RMS_values)
        plt.show()
    
        
    if Z_deviations is not None:
        RMS_values = np.empty( len(Z_deviations) )
        
        first_source.guess_XYZT[:] = original_XYZT
        for i in range(len(Z_deviations)):
        
            first_source.guess_XYZT[2] = original_XYZT[2] + Z_deviations[i]
            fitter = stochastic_fitter_dt(current_sources)
            RMS_values[i] = fitter.RMS
            
            print('Z', Z_deviations[i], fitter.RMS)
            
        plt.plot(Z_deviations, RMS_values)
        plt.show()
コード例 #16
0
 def open_files(self, station, log_func=do_nothing, force_metadata_ant_pos=True):
     processed_data_loc = processed_data_dir(self.timeID)
     
     #### open callibration data files ####
     ### TODO: fix these so they are accounted for in a more standard maner
     self.additional_ant_delays = read_antenna_delays( processed_data_loc+'/'+self.additional_antenna_delays_fname )
     self.bad_antennas = read_bad_antennas( processed_data_loc+'/'+self.bad_antennas_fname )
     self.pol_flips = read_antenna_pol_flips( processed_data_loc+'/'+self.pol_flips_fname )
     
     #### open data files, and find RFI ####
     self.station = station
     raw_fpaths = filePaths_by_stationName(self.timeID)[station]
     self.input_file = MultiFile_Dal1( raw_fpaths, force_metadata_ant_pos=force_metadata_ant_pos )
     self.input_file.set_polarization_flips( self.pol_flips )
     
     if self.do_RFI_filtering and self.use_saved_RFI_info:
         self.RFI_filters.append( window_and_filter(timeID=self.timeID, sname=station) )
     
     elif self.do_RFI_filtering:
         RFI_result = FindRFI(self.input_file, self.block_size, self.initial_RFI_block, self.RFI_num_blocks, self.RFI_max_blocks, verbose=False, figure_location=None)
         self.RFI_filters.append( window_and_filter(find_RFI=RFI_result) )
         
     else: ## only basic filtering
         self.RFI_filters.append( window_and_filter(blocksize=self.block_size) )
     
     
     
     #### find antenna pairs ####
     self.antennas_to_use = pairData_evenAnts(self.input_file, self.bad_antennas, self.max_num_antennas )
     self.num_antennas = len(self.antennas_to_use)
     
     
     #### get antenna locations and delays ####
     self.antenna_locations = np.zeros((self.num_antennas, 3), dtype=np.double)
     self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double)
     
     for ant_i, station_ant_i in enumerate(self.antennas_to_use):
         
         ant_name = self.input_file.get_antenna_names()[ station_ant_i ]
         
         self.antenna_locations[ant_i] = self.input_file.get_LOFAR_centered_positions()[ station_ant_i ]
         self.antenna_delays[ant_i] = self.input_file.get_timing_callibration_delays()[ station_ant_i ]
         
         ## add additional timing delays
         ##note this only works for even antennas!
         if ant_name in self.additional_ant_delays:
             self.antenna_delays[ant_i] += self.additional_ant_delays[ ant_name ][0]
     
     
     
     ##### find window offsets and lengths ####
     self.half_antenna_data_length = np.zeros(self.num_antennas, dtype=np.long)
             
     for ant_i, station_ant_i in enumerate(self.antennas_to_use):
         
         ### find max duration to any of the prefered antennas
         max_distance = 0.0
         for ant_j, station_ant_j in enumerate(self.antennas_to_use):
             if ant_j == ant_i:
                 continue
             
             distance = np.linalg.norm( self.antenna_locations[ ant_j ]-self.antenna_locations[ant_i] )
             if distance > max_distance:
                 max_distance = distance
                 
         self.half_antenna_data_length[ant_i] = int(self.pulse_length/2) + int(max_distance/(v_air*5.0E-9))
     
     self.block_exclusion_length = int(0.1*self.block_size) + np.max( self.half_antenna_data_length ) + 1
     self.active_block_length = self.block_size - 2*self.block_exclusion_length##the length of block used for looking at data
     
     self.trace_length = 2*np.max(self.half_antenna_data_length )
     self.trace_length = 2**( int(np.log2( self.trace_length )) + 1 )
     
     
     #### allocate some memory ####
     self.data_block = np.empty((self.num_antennas,self.block_size), dtype=np.complex)
     self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double)
     
     self.stage_1_imager = II_tools.image_data(self.antenna_locations, self.antenna_delays, self.trace_length, self.upsample_factor)
コード例 #17
0
#!/usr/bin/env python3

from LoLIM.IO.raw_tbb_IO import MultiFile_Dal1, filePaths_by_stationName

## these lines are anachronistic and should be fixed at some point
from LoLIM import utilities
utilities.default_raw_data_loc = "/exp_app2/appexp1/public/raw_data"
utilities.default_processed_data_loc = "/home/brian/processed_files"

if __name__ == "__main__":
    
    timeID = "D20170929T202255.000Z"
    
    raw_fpaths = filePaths_by_stationName(timeID)
    TBB_data = MultiFile_Dal1( raw_fpaths["CS002"] )
    print(TBB_data.get_timestamp())
        
        
コード例 #18
0
    def __init__(self,
                 timeID,
                 guess_delays,
                 block_size,
                 initial_block,
                 num_blocks,
                 working_folder,
                 other_folder=None,
                 max_num_stations=np.inf,
                 guess_location=None,
                 bad_stations=[],
                 polarization_flips="polarization_flips.txt",
                 bad_antennas="bad_antennas.txt",
                 additional_antenna_delays=None,
                 do_remove_saturation=True,
                 do_remove_RFI=True,
                 positive_saturation=2046,
                 negative_saturation=-2047,
                 saturation_post_removal_length=50,
                 saturation_half_hann_length=50,
                 referance_station="CS002",
                 pulse_length=50,
                 upsample_factor=4,
                 min_antenna_amplitude=5,
                 fix_polarization_delay=True):

        #### disable matplotlib shortcuts
        plt.rcParams['keymap.save'] = ''
        plt.rcParams['keymap.fullscreen'] = ''
        plt.rcParams['keymap.home'] = ''
        plt.rcParams['keymap.back'] = ''
        plt.rcParams['keymap.forward'] = ''
        plt.rcParams['keymap.pan'] = ''
        plt.rcParams['keymap.zoom'] = ''
        plt.rcParams['keymap.save'] = ''
        plt.rcParams['keymap.quit_all'] = ''
        plt.rcParams['keymap.grid'] = ''
        plt.rcParams['keymap.grid_minor'] = ''
        plt.rcParams['keymap.yscale'] = ''
        plt.rcParams['keymap.xscale'] = ''
        plt.rcParams['keymap.all_axes'] = ''
        plt.rcParams['keymap.copy'] = ''

        #keymap.help : f1                    ## display help about active tools
        #keymap.quit : ctrl+w, cmd+w, q      ## close the current figure

        self.pressed_data = None
        #guess_location = np.array(guess_location[:3])

        self.block_size = block_size
        self.num_blocks = num_blocks

        self.positive_saturation = positive_saturation
        self.negative_saturation = negative_saturation
        self.saturation_post_removal_length = saturation_post_removal_length
        self.saturation_half_hann_length = saturation_half_hann_length
        self.do_remove_saturation = do_remove_saturation
        self.do_remove_RFI = do_remove_RFI
        self.referance_station = referance_station

        #### open data ####
        processed_data_folder = processed_data_dir(timeID)
        polarization_flips = read_antenna_pol_flips(processed_data_folder +
                                                    '/' + polarization_flips)
        bad_antennas = read_bad_antennas(processed_data_folder + '/' +
                                         bad_antennas)
        if additional_antenna_delays is not None:
            additional_antenna_delays = read_antenna_delays(
                processed_data_folder + '/' + additional_antenna_delays)
            print(
                "WARNING: Additional antenna delays should probably be None!")

        raw_fpaths = filePaths_by_stationName(timeID)
        station_names = [
            sname for sname in raw_fpaths.keys() if sname not in bad_stations
        ]
        self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
                          for sname in station_names}

        if fix_polarization_delay:
            for sname, file in self.TBB_files.items():

                #                delays = file.get_timing_callibration_delays()
                #                print()
                #                print()
                #                print(sname)
                #                for even_ant_i in range(0,len(delays),2):
                #                    print("  ", delays[even_ant_i+1]-delays[even_ant_i])

                file.find_and_set_polarization_delay()

#                delays = file.get_timing_callibration_delays()

#                print("after")
#                for even_ant_i in range(0,len(delays),2):
#                    print("  ", delays[even_ant_i+1]-delays[even_ant_i])

        self.RFI_filters = {
            sname: window_and_filter(timeID=timeID, sname=sname)
            for sname in station_names
        }
        self.edge_width = int(self.block_size * 0.1)
        self.display_block_size = self.block_size - 2 * self.edge_width

        #### some data things ####
        self.antenna_time_corrections = {
            sname: TBB_file.get_total_delays()
            for sname, TBB_file in self.TBB_files.items()
        }
        self.ave_ref_ant_delay = np.average(
            self.antenna_time_corrections[referance_station])

        #        ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0]
        #        for station_corrections in self.antenna_time_corrections.values():
        #            station_corrections -= ref_antenna_delay

        max_num_antennas = np.max(
            [len(delays) for delays in self.antenna_time_corrections.values()])
        self.temp_data_block = np.empty(
            (max_num_antennas, num_blocks, block_size), dtype=np.double)
        self.time_array = np.arange(block_size) * 5.0E-9

        self.figure = plt.gcf()
        self.axes = plt.gca()

        #### make managers
        guess_delays = {
            sname: delay
            for sname, delay in guess_delays.items()
            if sname not in bad_stations
        }
        self.clock_offset_manager = clock_offsets(referance_station,
                                                  guess_delays, guess_location,
                                                  self.TBB_files)
        self.frame_manager = frames(initial_block, int(num_blocks / 2),
                                    station_names, max_num_stations,
                                    referance_station, self.display_block_size,
                                    num_blocks)

        rem_sat_curry = cur(remove_saturation, 5)(
            positive_saturation=positive_saturation,
            negative_saturation=negative_saturation,
            post_removal_length=saturation_post_removal_length,
            half_hann_length=saturation_half_hann_length)
        self.pulse_manager = pulses(
            pulse_length=pulse_length,
            block_size=block_size,
            out_folder=working_folder,
            other_folder=other_folder,
            TBB_data_dict=self.TBB_files,
            used_delay_dict=self.antenna_time_corrections,
            upsample_factor=upsample_factor,
            min_ant_amp=min_antenna_amplitude,
            referance_station=referance_station,
            RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None),
            remove_saturation_curry=(rem_sat_curry
                                     if self.do_remove_saturation else None))

        self.plot()

        self.mode = 0  ##0 is change delay, 1 is set peak, etc...
        self.input_mode = 0  ## 0 is normal. press 's' to enter 1, where numbers are used to enter event. press 's' again to go back to 0, and search for event
        self.search_string = ''
        self.rect_selector = RectangleSelector(plt.gca(),
                                               self.rectangle_callback,
                                               useblit=True,
                                               rectprops=dict(alpha=0.5,
                                                              facecolor='red'),
                                               button=1,
                                               state_modifier_keys={
                                                   'move': '',
                                                   'clear': '',
                                                   'square': '',
                                                   'center': ''
                                               })

        self.mouse_press = self.figure.canvas.mpl_connect(
            'button_press_event', self.mouse_press)
        self.mouse_release = plt.gcf().canvas.mpl_connect(
            'button_release_event', self.mouse_release)

        self.key_press = plt.gcf().canvas.mpl_connect('key_press_event',
                                                      self.key_press_event)

        self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()]

        plt.show()
コード例 #19
0
def plot_stations_AllData(timeID,
                          input_file_name,
                          source_XYZT,
                          known_station_delays,
                          stations="all",
                          referance_station="CS002",
                          min_antenna_amplitude=10,
                          skip_stations=[],
                          antennas_to_exclude=[],
                          seperation_factor=1):
    """Plot all pulses on all stations in an autocorrelator file. polarization should be 0 or 1. stations should be "all", "RS", or "CS" """

    if referance_station not in known_station_delays:
        known_station_delays[referance_station] = 0
    known_station_delays = {
        station: T - known_station_delays[referance_station]
        for station, T in known_station_delays.items()
    }

    data_file = h5py.File(input_file_name, "r")

    station_names = [
        sname for sname in data_file.keys() if sname not in skip_stations
    ]
    known_station_delays = {
        sname:
        known_station_delays[sname] if sname in known_station_delays else 0.0
        for sname in station_names
    }

    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {
        sname: MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True)
        for sname in station_names
    }

    stat_i = 0
    for (sname, stat_group) in data_file.items():
        if (stations != "all"
                and stations != sname[:2]) or sname in skip_stations:
            continue

        stat_data = raw_data_files[sname]
        ant_loc_dict = {
            ant_name: ant_loc
            for ant_name, ant_loc in zip(
                stat_data.get_antenna_names(),
                stat_data.get_LOFAR_centered_positions())
        }

        trace_max = 0.0
        for ant_name, ant_dataset in stat_group.items():
            trace = np.array(ant_dataset[1])
            if np.max(trace) > trace_max:
                trace_max = np.max(trace)
            trace = np.array(ant_dataset[3])
            if np.max(trace) > trace_max:
                trace_max = np.max(trace)

        trace_max *= 2

        for ant_name, ant_dataset in stat_group.items():

            even_trace = np.array(ant_dataset[0])
            even_HE = np.array(ant_dataset[1])
            odd_trace = np.array(ant_dataset[2])
            odd_HE = np.array(ant_dataset[3])
            trace_amp = max(np.max(even_HE), np.max(odd_HE))

            if trace_amp < min_antenna_amplitude or ant_name in antennas_to_exclude:
                continue

            ant_loc = ant_loc_dict[ant_name]
            travel_delay = np.sqrt((ant_loc[0] - source_XYZT[0])**2 +
                                   (ant_loc[1] - source_XYZT[1])**2 +
                                   (ant_loc[2] - source_XYZT[2])**2) / v_air
            total_correction = travel_delay + source_XYZT[
                3] + known_station_delays[sname]

            step = stat_i * seperation_factor

            even_peak_time = ant_dataset.attrs[
                'PolE_timeOffset'] - total_correction
            odd_peak_time = ant_dataset.attrs[
                'PolO_timeOffset'] - total_correction

            trace_time = np.arange(len(
                trace)) * 5.0E-9 + ant_dataset.attrs['starting_index'] * 5.0E-9
            even_trace_time = trace_time + even_peak_time
            odd_trace_time = trace_time + odd_peak_time

            plt.plot(even_trace_time, even_trace / trace_max + step, color='g')
            plt.plot(even_trace_time, even_HE / trace_max + step, color='g')

            plt.plot(odd_trace_time, odd_trace / trace_max + step, color='m')
            plt.plot(odd_trace_time, odd_HE / trace_max + step, color='m')

        stat_i += 1

        plt.annotate(sname, xy=(0.0, step + 0.5))
    plt.axvline(x=0)
    plt.show()
コード例 #20
0
def plot_blocks(timeID, block_size, block_starts, guess_delays, guess_location = None, bad_stations=[], polarization_flips="polarization_flips.txt",
                bad_antennas = "bad_antennas.txt", additional_antenna_delays = "ant_delays.txt", do_remove_saturation = True, do_remove_RFI = True,
                positive_saturation = 2046, negative_saturation = -2047, saturation_post_removal_length = 50, saturation_half_hann_length = 5,
                referance_station = "CS002", amplify = {}, omitPOL=None):
    """plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays,
    if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero.
    A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations"""

    '''Modified version of plot_multiple_data_all_stations to support usage of clickable GUI for aligning station timings'''


##2
    global first_few_names
    global defualt_color
    global ax

    text = []
##~2

    if referance_station in guess_delays:
        ref_delay = guess_delays[referance_station]
        guess_delays = {sname:delay-ref_delay for sname,delay in guess_delays.items()}

    processed_data_folder = processed_data_dir(timeID)

    polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
    bad_antennas = read_bad_antennas( processed_data_folder + '/' + bad_antennas )
    additional_antenna_delays = read_antenna_delays(  processed_data_folder + '/' + additional_antenna_delays )


    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
                      for sname in raw_fpaths.keys() if sname not in bad_stations}

    if guess_location is not None:
        guess_location = np.array(guess_location)

        ref_stat_file = raw_data_files[ referance_station ]
        ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
        ref_delay = np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_stat_file.get_nominal_sample_number()*5.0E-9

        for sname, data_file in raw_data_files.items():
            if sname not in guess_delays:
                guess_delays[sname] = 0.0

            data_file = raw_data_files[sname]
            ant_loc = data_file.get_LOFAR_centered_positions()[0]
            guess_delays[sname] += (np.linalg.norm(ant_loc-guess_location[:3])/v_air - ref_delay)
            guess_delays[sname] -= data_file.get_nominal_sample_number()*5.0E-9

    RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in raw_fpaths.keys() if sname not in bad_stations}
    #RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname, blocksize=block_size) for sname in raw_fpaths.keys() if sname not in bad_stations}

    data = np.empty(block_size, dtype=np.double)


##3
    #fig = plt.figure(figsize=(10, 12))
    fig, ax = plt.subplots()

    for sname, data_file in raw_data_files.items():
        #print("Loading: "+ sname)
        if referance_station in sname:
           ref_plot = (sname, data_file)
           #ref_plot_offset = guess_delays[sname] #currently unsed
        else:
           plots.append((sname, data_file))


    first_few = [ref_plot]
    for p in range(0,Nplots-1):
        first_few.append(plots.pop(0))


    for sname, data_file in first_few:
        if sname != referance_station:
           first_few_names.append(sname)
##~3

##4
    def more_plots(some_data,First,ax,text):
       global l,m
       ax.clear()

       height = 0
       t0 = np.arange(block_size)*5.0E-9
       transform = blended_transform_factory(plt.gca().transAxes, plt.gca().transData)
       sname_X_loc = 0.0
       n = 0
       for sname, data_file in some_data:

        print("Plotting: "+sname)

        station_delay = -data_file.get_nominal_sample_number()*5.0E-9
        if sname in guess_delays:
            station_delay = guess_delays[sname]

        station_delay_points = int(station_delay/5.0E-9)

        RFI_filter = RFI_filters[sname]
        ant_names = data_file.get_antenna_names()

        num_antenna_pairs = int( len( ant_names )/2 )
        peak_height = 0.0

        print("Block starts at: "+str(block_starts[0]))
        for point in block_starts:
            T = t0 + point*5.0E-9
            for pair in range(num_antenna_pairs):
                data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2)

                if do_remove_saturation:
                    remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
                if do_remove_RFI:
                    filtered_data = RFI_filter.filter( data )
                else:
                    filtered_data = hilbert(data)
                even_HE = np.abs(filtered_data)

                data[:] = data_file.get_data(point+station_delay_points, block_size, antenna_index=pair*2+1)
                if do_remove_saturation:
                    remove_saturation(data, positive_saturation, negative_saturation, saturation_post_removal_length, saturation_half_hann_length)
                if do_remove_RFI:
                    filtered_data = RFI_filter.filter( data )
                else:
                    filtered_data = hilbert(data)
                odd_HE = np.abs(filtered_data)

                #ax.plot(T, even_HE + height, 'r')
                #ax.plot(T, odd_HE + height, 'g' )

                #for cases where data are hard to align due to signal being tiny:
                found = False
                if len(amplify) > 0:
                    for key in amplify:
                        if sname == key:

                            if omitPOL != 0:
                               ax.plot(T, amplify[key]*even_HE + height, 'r')
                            if omitPOL != 1:
                               ax.plot(T, amplify[key]*odd_HE + height, 'g' )

                            #ax.plot(T, amplify[key]*even_HE + height, 'r') twinx with block_starts && point as x rather than t!  (can I update to latests plot_multiple?)
                            #ax.plot(T, amplify[key]*odd_HE + height, 'g' )

                            found = True
                if not found:
                    if omitPOL != 0:
                       ax.plot(T, even_HE + height, 'r')
                    if omitPOL != 1:
                       ax.plot(T, odd_HE + height, 'g' )
                    #ax2 = ax.twiny()
                    #xmin = min(block_starts)
                    #xmax = max(block_starts)
                    #ax2.set_xlim(xmin, xmax) #can also use ax2.set_xticks(x)
                    #ax2.xaxis.set_ticks_position('both')

                max_even = np.max(even_HE)
                if max_even > peak_height:
                    peak_height = max_even
                max_odd = np.max(odd_HE)
                if max_odd > peak_height:
                    peak_height = max_odd

#        plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
        plt.sca(ax) #somehow this makes it so that the annotations work on all plots except for the first time "Next stations" is clicked
        plt.annotate(sname, (sname_X_loc, height), textcoords=transform, xycoords=transform)

        ax.ticklabel_format(useOffset=False)
        plt.setp(ax.get_xticklabels(), rotation=60, horizontalalignment='right')
        plt.subplots_adjust(bottom=0.1,top=0.98,left=0.21)
        if ref_plot_peak_location != 0.0:
           print("Setting ref peak to " +str(ref_plot_peak_location))
           Cpeak = ref_plot_peak_location #peak center, for plotting
           Dpeak = 0.0001  # +/- added to peak center for plotting
           plt.xlim(Cpeak-Dpeak,Cpeak+Dpeak)
        plt.grid(b=True)

        height += 2*peak_height
        n+=1
##~4





##5
#######################CallBacks############################
#######################CallBacks############################
#######################CallBacks############################
    def onclick(event):
        global stations_active
        global station_buttons
        global ref_plot_peak_location
        global set_ref
        global set_stations
        global default_color
        ix = event.xdata
        iy = event.ydata
        #print("click active stations: ", stations_active)
        falsify = []

        if set_ref:
           ref_plot_peak_location = ix
           print("Set ref peak to: "+str(ref_plot_peak_location))
           set_ref = False
           return #return, don't set anything else!
        n=0
        for s in stations_active.keys():
          if stations_active[s] and s !="":
            falsify.append(s)
            if s not in guess_delays.keys():
                guess_delays[s] = 0.0
            print ("Set " + s + " station delay to: ", guess_delays[s] + (ix-ref_plot_peak_location))
            tmp_guess_delays[s] = guess_delays[s] + (ix - ref_plot_peak_location)

            station_buttons[n].color = default_color
            station_buttons[n].hovercolor = default_color

          n+=1

        if len(falsify)>0:
           for f in falsify:
               stations_active[f] = False
           falsify = []
        plt.draw()


    def nextCB(event):
        global ax
        global plots
        global station_buttons
        global stations_active
        global StationCallBacks
        print("next!")

        stations_active = {}
        next_few = [ref_plot]
        if len(plots) >= Nplots:
         for i in range(0,Nplots-1):
               next_few.append(plots.pop(0))
        elif len(plots) > 0:
         bk = len(plots)
         for i in range(0,len(plots)):
               next_few.append(plots.pop(0))
         for i in range(bk,4):
             StationCallBacks[i].reset(i,"")
             station_buttons[i].label.set_text("")

        else:
          next_few = []
          for i in range(0,Nplots-1):
             StationCallBacks[i].reset(i,"")
             station_buttons[i].label.set_text("")
          print("All stations timings have been set")
          ax.clear()
          ax.annotate("All stations timings have been set", (0.5, 0.5))
          plt.draw()
          return

        height = 0.0
        n = 0
        for sname, data_file in next_few:

            if sname != referance_station:
               first_few_names.append(sname)
               StationCallBacks[n].reset(n,sname)
               station_buttons[n].label.set_text("Set "+sname)
               stations_active[sname] = False
               n+=1

        more_plots(next_few,False,ax,text)
        plt.draw()


    def refpCB(event):
        global ref_plot_peak_location
        global set_ref
        print ("Setting ref peak")# to: ",ix)
        set_ref = True

    def cCB(text):
        Cpeak = float(text) #peak center, for plotting
        Dpeak = 0.0001  # +/- added to peak center for plotting
        plt.xlim(Cpeak+Dpeak,Cpeak-Dpeak)
        plt.draw()

    def writeCB(event):
        print('Writing guess station delays')
        file = open('guess_delays.py','w')
        file.write("\n      self.guess_timing = " +str(ref_plot_peak_location) + "\n")
        #file.write("      self.event_index = ____\n\n")
        file.write('      self.guess_station_delays = {\n')
        for g in guess_delays:
            if g in tmp_guess_delays:
               '''If the delay has been updated, write the update:'''
               file.write('"'+g+'": '+str(tmp_guess_delays[g])+',\n')
            else:
               """Else, just write the old delay:"""
               file.write('"'+g+'": '+str(guess_delays[g])+',\n')
        file.write('}\n')
        print('Done!\n')


    def press(event):
        val = event.key
        print('press', val)
        '''if val == ('0' or  '1' or '2' or '3' or '4'):
           val = int(val)
           station_buttons[val].color = 'blue'
           station_buttons[val] .hovercolor = 'blue'
        '''


    class StationCB:
        global stations_active
        def __init__(self,np,station):
            self.N = np
            self.station = station

        def reset(self,np,station):
            self.N = np
            self.station = station

        def stationCB(self,event):
           print("Station active: "+self.station)
           #print(self.station + ' You pressed: ',event.name)
           #---highlight button with color when pressed state, show buttons in correct order
           station_buttons[self.N].color = 'blue'
           station_buttons[self.N].hovercolor = 'blue'
           stations_active[self.station] = True
######################~CallBacks############################
######################~CallBacks############################
######################~CallBacks############################

    more_plots(first_few,True,ax,text)

    axnext = plt.axes([ 0.05, 0.80,  0.1, 0.07])
    bnext = Button(axnext, 'Next\nStations')
    bnext.on_clicked(nextCB)

    axfile = plt.axes([0.05,0.7,  0.1,  0.07])
    bfile = Button(axfile, 'Write\nGuesses')
    bfile.on_clicked(writeCB)

    axref = plt.axes([0.05, 0.1,  0.13, 0.07])
    bref= Button(axref, 'Set\nRef Peak')
    bref.on_clicked(refpCB)

    #Cbox = plt.axes([0.05, 0.6,  0.1, 0.07])
    #text_box = TextBox(Cbox, 'Peak Center', initial="")
    #text_box.on_submit(cCB)

    start = 0.22
    for f in range(0,Nplots-1):
        fax = plt.axes([ 0.05, start+f*0.1,  0.13, 0.05])
        fbtn = Button(fax, "Set "+first_few_names[f])
        stations_active[first_few_names[f]] = False
        s = StationCB(f,first_few_names[f])
        StationCallBacks.append(s)
        station_axes.append(fax)
        station_buttons.append(fbtn)
        fbtn.on_clicked(s.stationCB)

    fig.canvas.mpl_connect('button_press_event', onclick)
    fig.canvas.mpl_connect('key_press_event', press)

    plt.show()
コード例 #21
0
def plot_one_station(timeID,
                     polarization,
                     input_file_name,
                     source_XYZT,
                     known_station_delays,
                     station,
                     referance_station="CS002",
                     min_antenna_amplitude=10,
                     plot_real=False,
                     antennas_to_exclude=[]):
    """Plot all pulses on all stations in an autocorrelator file. polarization should be 0 or 1. station should be station name """

    if polarization == 0:
        dataset_index = 1  ##1 for even HE, 3 for odd HE
        peaktime_pol = "PolE_peakTime"  ## even or odd peak times
        antOffset_pol = 'PolE_timeOffset'
    elif polarization == 1:
        dataset_index = 3  ##1 for even HE, 3 for odd HE
        peaktime_pol = "PolO_peakTime"  ## even or odd peak times
        antOffset_pol = 'PolO_timeOffset'

    if referance_station not in known_station_delays:
        known_station_delays[referance_station] = 0
    known_station_delays = {
        station: T - known_station_delays[referance_station]
        for station, T in known_station_delays.items()
    }

    data_file = h5py.File(input_file_name, "r")

    station_names = list(data_file.keys())
    known_station_delays = {
        sname:
        known_station_delays[sname] if sname in known_station_delays else 0.0
        for sname in station_names
    }

    raw_fpaths = filePaths_by_stationName(timeID)
    stat_group = data_file[station]
    stat_data = MultiFile_Dal1(raw_fpaths[station],
                               force_metadata_ant_pos=True)

    current_amp = 0.0
    for ant_name, ant_loc in zip(stat_data.get_antenna_names(),
                                 stat_data.get_LOFAR_centered_positions()):
        if ant_name not in stat_group:
            continue

        ant_dataset = stat_group[ant_name]
        trace = np.array(ant_dataset[dataset_index])
        trace_amp = np.max(trace)

        if trace_amp < min_antenna_amplitude or ant_name in antennas_to_exclude:
            continue

        travel_delay = np.sqrt((ant_loc[0] - source_XYZT[0])**2 +
                               (ant_loc[1] - source_XYZT[1])**2 +
                               (ant_loc[2] - source_XYZT[2])**2) / v_air
        total_correction = travel_delay + source_XYZT[
            3] + known_station_delays[station]

        peak_time = ant_dataset.attrs[peaktime_pol] - total_correction

        p = plt.plot((peak_time, peak_time),
                     (current_amp, current_amp + trace_amp))
        print(ant_name, peak_time, ant_dataset.attrs[antOffset_pol])

        trace_time = np.arange(len(trace)) * 5.0E-9 + ant_dataset.attrs[
            'starting_index'] * 5.0E-9 + ant_dataset.attrs[
                antOffset_pol] - total_correction
        plt.plot(trace_time, trace + current_amp, color=p[0].get_color())

        if plot_real:
            rt = np.array(ant_dataset[dataset_index - 1])
            plt.plot(trace_time, rt + current_amp, color=p[0].get_color())

        plt.annotate(ant_name, xy=(0.0, current_amp + 0.5 * trace_amp))

        current_amp += trace_amp
    plt.axvline(x=0)
    plt.show()
コード例 #22
0
    def __init__(self,
                 timeID,
                 guess_delays,
                 block_size,
                 initial_block,
                 num_blocks,
                 working_folder,
                 other_folder=None,
                 max_num_stations=np.inf,
                 guess_location=None,
                 bad_stations=[],
                 polarization_flips="polarization_flips.txt",
                 bad_antennas="bad_antennas.txt",
                 additional_antenna_delays="ant_delays.txt",
                 do_remove_saturation=True,
                 do_remove_RFI=True,
                 positive_saturation=2046,
                 negative_saturation=-2047,
                 saturation_post_removal_length=50,
                 saturation_half_hann_length=50,
                 referance_station="CS002",
                 pulse_length=50,
                 upsample_factor=4,
                 min_antenna_amplitude=5,
                 fix_polarization_delay=True):

        self.pressed_data = None
        #guess_location = np.array(guess_location[:3])

        self.block_size = block_size
        self.num_blocks = num_blocks

        self.positive_saturation = positive_saturation
        self.negative_saturation = negative_saturation
        self.saturation_post_removal_length = saturation_post_removal_length
        self.saturation_half_hann_length = saturation_half_hann_length
        self.do_remove_saturation = do_remove_saturation
        self.do_remove_RFI = do_remove_RFI

        #### open data ####
        processed_data_folder = processed_data_dir(timeID)
        polarization_flips = read_antenna_pol_flips(processed_data_folder +
                                                    '/' + polarization_flips)
        bad_antennas = read_bad_antennas(processed_data_folder + '/' +
                                         bad_antennas)
        additional_antenna_delays = read_antenna_delays(
            processed_data_folder + '/' + additional_antenna_delays)

        raw_fpaths = filePaths_by_stationName(timeID)
        station_names = [
            sname for sname in raw_fpaths.keys() if sname not in bad_stations
        ]
        self.TBB_files = {sname:MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
                          for sname in station_names}

        if fix_polarization_delay:
            for sname, file in self.TBB_files.items():

                #                delays = file.get_timing_callibration_delays()
                #                print()
                #                print()
                #                print(sname)
                #                for even_ant_i in range(0,len(delays),2):
                #                    print("  ", delays[even_ant_i+1]-delays[even_ant_i])

                file.find_and_set_polarization_delay()

#                delays = file.get_timing_callibration_delays()

#                print("after")
#                for even_ant_i in range(0,len(delays),2):
#                    print("  ", delays[even_ant_i+1]-delays[even_ant_i])

#self.RFI_filters = {sname:window_and_filter(timeID=timeID,sname=sname) for sname in station_names}
        self.RFI_filters = {
            sname: window_and_filter(timeID=timeID,
                                     sname=sname,
                                     blocksize=block_size)
            for sname in station_names
        }

        #### some data things ####
        self.antenna_time_corrections = {
            sname: TBB_file.get_total_delays()
            for sname, TBB_file in self.TBB_files.items()
        }
        #        ref_antenna_delay = self.antenna_time_corrections[ referance_station ][0]
        #        for station_corrections in self.antenna_time_corrections.values():
        #            station_corrections -= ref_antenna_delay

        max_num_antennas = np.max(
            [len(delays) for delays in self.antenna_time_corrections.values()])
        self.temp_data_block = np.empty(
            (max_num_antennas, num_blocks, block_size), dtype=np.double)
        self.time_array = np.arange(block_size) * 5.0E-9

        self.figure = plt.gcf()
        self.axes = plt.gca()

        #### make managers
        guess_delays = {
            sname: delay
            for sname, delay in guess_delays.items()
            if sname not in bad_stations
        }
        self.clock_offset_manager = clock_offsets(referance_station,
                                                  guess_delays, guess_location,
                                                  self.TBB_files)
        self.frame_manager = frames(initial_block, int(num_blocks / 2),
                                    station_names, max_num_stations,
                                    referance_station, block_size, num_blocks)

        rem_sat_curry = cur(remove_saturation, 5)(
            positive_saturation=positive_saturation,
            negative_saturation=negative_saturation,
            post_removal_length=saturation_post_removal_length,
            half_hann_length=saturation_half_hann_length)
        self.pulse_manager = pulses(
            pulse_length=pulse_length,
            block_size=block_size,
            out_folder=working_folder,
            other_folder=other_folder,
            TBB_data_dict=self.TBB_files,
            used_delay_dict=self.antenna_time_corrections,
            upsample_factor=upsample_factor,
            min_ant_amp=min_antenna_amplitude,
            referance_station=referance_station,
            RFI_filter_dict=(self.RFI_filters if self.do_remove_RFI else None),
            remove_saturation_curry=(rem_sat_curry
                                     if self.do_remove_saturation else None))

        self.plot()

        self.mode = 0  ##0 is change delay, 1 is set peak
        self.rect_selector = RectangleSelector(plt.gca(),
                                               self.rectangle_callback,
                                               useblit=True,
                                               rectprops=dict(alpha=0.5,
                                                              facecolor='red'),
                                               button=1,
                                               state_modifier_keys={
                                                   'move': '',
                                                   'clear': '',
                                                   'square': '',
                                                   'center': ''
                                               })

        self.mouse_press = self.figure.canvas.mpl_connect(
            'button_press_event', self.mouse_press)
        self.mouse_release = plt.gcf().canvas.mpl_connect(
            'button_release_event', self.mouse_release)

        self.key_press = plt.gcf().canvas.mpl_connect('key_press_event',
                                                      self.key_press_event)

        self.home_lims = [self.axes.get_xlim(), self.axes.get_ylim()]

        plt.show()
コード例 #23
0
additional_antenna_delays = "ant_delays.txt"

polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
                                            polarization_flips)
bad_antennas = read_bad_antennas(processed_data_folder + '/' + bad_antennas)
additional_antenna_delays = read_antenna_delays(processed_data_folder + '/' +
                                                additional_antenna_delays)
station_timing_offsets = read_station_delays(processed_data_folder + '/' +
                                             station_delay_file)

raw_fpaths = filePaths_by_stationName(timeID)

TBB_data = {
    sname: MultiFile_Dal1(fpath,
                          force_metadata_ant_pos=True,
                          polarization_flips=polarization_flips,
                          bad_antennas=bad_antennas,
                          additional_ant_delays=additional_antenna_delays,
                          only_complete_pairs=True)
    for sname, fpath in raw_fpaths.items() if sname in station_timing_offsets
}


def model(p, r):
    ϕ = np.arctan2(r[:, 1], r[:, 0])
    ϕ_hat = np.array([-np.sin(ϕ), np.cos(ϕ),
                      np.zeros(ϕ.size)]).T  #unit azimuth vector

    r_hat = r / np.linalg.norm(
        r, axis=1)[:, np.newaxis]  #radial unit vector to source

    a = np.array([
コード例 #24
0
    additional_antenna_delays = "ant_delays.txt"

    processed_data_folder = processed_data_dir(timeID)

    polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
                                                polarization_flips)
    bad_antennas = read_bad_antennas(processed_data_folder + '/' +
                                     bad_antennas)
    additional_antenna_delays = read_antenna_delays(processed_data_folder +
                                                    '/' +
                                                    additional_antenna_delays)
    station_timing_offsets = read_station_delays(processed_data_folder + '/' +
                                                 station_delay_file)

    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {sname:MultiFile_Dal1(fpaths, force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
                      for sname,fpaths in raw_fpaths.items() if sname in station_timing_offsets}

    data_filters = {
        sname: window_and_filter(timeID=timeID, sname=sname)
        for sname in station_timing_offsets
    }

    trace_locator = getTrace_fromLoc(raw_data_files, data_filters,
                                     station_timing_offsets)

    interferometry_header, IPSE_list = R_IPSE.load_interferometric_PSE(
        processed_data_folder + "/" + input_folder,
        blocks_to_open=[IPSE_block])
    IPSE = [IPSE for IPSE in IPSE_list
            if IPSE.unique_index == IPSE_unique_ID][0]
コード例 #25
0
def recalibrate_pulse(timeID, input_fname, output_fname, set_polarization_delay=True, upsample_factor=4, polarization_flips="polarization_flips.txt"):

    processed_data_folder = processed_data_dir(timeID)
    raw_fpaths = filePaths_by_stationName(timeID)
    polarization_flips = read_antenna_pol_flips( processed_data_folder + '/' + polarization_flips )
    
    input_file = h5py.File(processed_data_folder+'/'+input_fname, "r")
    
    try:
        output_file = h5py.File(processed_data_folder+'/'+output_fname, "r+")
    except:
        output_file = h5py.File(processed_data_folder+'/'+output_fname, "w")
        
    sample_time = 5.0e-9
    if upsample_factor>1:
        sample_time /= upsample_factor
    
    for sname, h5_statGroup in input_file.items():
        out_statGroup = output_file.require_group(sname)
        print()
        print()
        print(sname)
        
        datafile = MultiFile_Dal1(raw_fpaths[sname], polarization_flips=polarization_flips)
        
        if set_polarization_delay:
            datafile.find_and_set_polarization_delay()
            
        antenna_calibrations = { antname:calibration for antname,calibration in zip(datafile.get_antenna_names(), datafile.get_total_delays()) }
        
        for antname, in_antData in h5_statGroup.items():
            out_statGroup.copy(in_antData,  out_statGroup, name= antname)
            out_antData = out_statGroup[antname]
            
            old_even =  out_antData.attrs['PolE_timeOffset_CS']
            old_odd = out_antData.attrs['PolO_timeOffset_CS']
            
            new_even_delay = antenna_calibrations[antname]
            new_odd_delay = antenna_calibrations[ even_antName_to_odd(antname) ]
            
            out_antData.attrs['PolE_timeOffset'] = -new_even_delay ## NOTE: due to historical reasons, there is a sign flip here
            out_antData.attrs['PolO_timeOffset'] = -new_odd_delay ## NOTE: due to historical reasons, there is a sign flip here
            out_antData.attrs['PolE_timeOffset_CS'] = new_even_delay
            out_antData.attrs['PolO_timeOffset_CS'] = new_odd_delay
            
            print(antname, old_even-new_even_delay, old_odd-new_odd_delay)
            
            starting_index = out_antData.attrs['starting_index']
            
            if np.isfinite( out_antData.attrs['PolE_peakTime'] ):
                
                polE_HE = out_antData[1]
                
                if upsample_factor>1:
                    polE_HE = resample(polE_HE, len(polE_HE)*upsample_factor )
            
                PolE_peak_finder = parabolic_fit( polE_HE  )
                out_antData.attrs['PolE_peakTime'] =  starting_index*5.0E-9 - new_even_delay + PolE_peak_finder.peak_index*sample_time
            
            if np.isfinite( out_antData.attrs['PolO_peakTime'] ):
                
                polO_HE = out_antData[3]
                
                if upsample_factor>1:
                    polO_HE = resample(polO_HE, len(polO_HE)*upsample_factor )
            
                PolO_peak_finder = parabolic_fit( polO_HE  )
                out_antData.attrs['PolO_peakTime'] =  starting_index*5.0E-9 - new_odd_delay + PolO_peak_finder.peak_index*sample_time
コード例 #26
0
def plot_blocks(timeID,
                block_size,
                block_starts,
                guess_delays,
                guess_location=None,
                bad_stations=[],
                polarization_flips="polarization_flips.txt",
                bad_antennas="bad_antennas.txt",
                additional_antenna_delays="ant_delays.txt",
                do_remove_saturation=True,
                do_remove_RFI=True,
                positive_saturation=2046,
                negative_saturation=-2047,
                saturation_post_removal_length=50,
                saturation_half_hann_length=5,
                referance_station="CS002"):
    """plot multiple blocks, for guessing initial delays and finding pulses. If guess_location is None, then guess_delays should be apparent delays,
    if guess_location is a XYZT location, then guess_delays should be real delays. If a station isn't in guess_delays, its' delay is assumed to be zero.
    A station is only not plotted if it is bad_stations. If referance station is in guess_delays, then its delay is subtract from all stations"""

    if referance_station in guess_delays:
        ref_delay = guess_delays[referance_station]
        guess_delays = {
            sname: delay - ref_delay
            for sname, delay in guess_delays.items()
        }

    processed_data_folder = processed_data_dir(timeID)

    polarization_flips = read_antenna_pol_flips(processed_data_folder + '/' +
                                                polarization_flips)
    bad_antennas = read_bad_antennas(processed_data_folder + '/' +
                                     bad_antennas)
    additional_antenna_delays = read_antenna_delays(processed_data_folder +
                                                    '/' +
                                                    additional_antenna_delays)

    raw_fpaths = filePaths_by_stationName(timeID)
    raw_data_files = {sname:MultiFile_Dal1(raw_fpaths[sname], force_metadata_ant_pos=True, polarization_flips=polarization_flips, bad_antennas=bad_antennas, additional_ant_delays=additional_antenna_delays) \
                      for sname in raw_fpaths.keys() if sname not in bad_stations}

    if guess_location is not None:
        guess_location = np.array(guess_location)

        ref_stat_file = raw_data_files[referance_station]
        ant_loc = ref_stat_file.get_LOFAR_centered_positions()[0]
        ref_delay = np.linalg.norm(
            ant_loc - guess_location[:3]
        ) / v_air - ref_stat_file.get_nominal_sample_number() * 5.0E-9

        for sname, data_file in raw_data_files.items():
            if sname not in guess_delays:
                guess_delays[sname] = 0.0

            data_file = raw_data_files[sname]
            ant_loc = data_file.get_LOFAR_centered_positions()[0]
            guess_delays[sname] += (
                np.linalg.norm(ant_loc - guess_location[:3]) / v_air -
                ref_delay)
            guess_delays[sname] -= data_file.get_nominal_sample_number(
            ) * 5.0E-9

    RFI_filters = {
        sname: window_and_filter(timeID=timeID, sname=sname)
        for sname in raw_fpaths.keys() if sname not in bad_stations
    }

    data = np.empty(block_size, dtype=np.double)

    height = 0
    t0 = np.arange(block_size) * 5.0E-9
    transform = blended_transform_factory(plt.gca().transAxes,
                                          plt.gca().transData)
    sname_X_loc = 0.0
    for sname, data_file in raw_data_files.items():
        print(sname)

        station_delay = -data_file.get_nominal_sample_number() * 5.0E-9
        if sname in guess_delays:
            station_delay = guess_delays[sname]

        station_delay_points = int(station_delay / 5.0E-9)

        RFI_filter = RFI_filters[sname]
        ant_names = data_file.get_antenna_names()

        num_antenna_pairs = int(len(ant_names) / 2)
        peak_height = 0.0
        for point in block_starts:
            T = t0 + point * 5.0E-9
            for pair in range(num_antenna_pairs):
                data[:] = data_file.get_data(point + station_delay_points,
                                             block_size,
                                             antenna_index=pair * 2)

                if do_remove_saturation:
                    remove_saturation(data, positive_saturation,
                                      negative_saturation,
                                      saturation_post_removal_length,
                                      saturation_half_hann_length)
                if do_remove_RFI:
                    filtered_data = RFI_filter.filter(data)
                else:
                    filtered_data = hilbert(data)
                even_HE = np.abs(filtered_data)

                data[:] = data_file.get_data(point + station_delay_points,
                                             block_size,
                                             antenna_index=pair * 2 + 1)
                if do_remove_saturation:
                    remove_saturation(data, positive_saturation,
                                      negative_saturation,
                                      saturation_post_removal_length,
                                      saturation_half_hann_length)
                if do_remove_RFI:
                    filtered_data = RFI_filter.filter(data)
                else:
                    filtered_data = hilbert(data)
                odd_HE = np.abs(filtered_data)

                plt.plot(T, even_HE + height, 'r')
                plt.plot(T, odd_HE + height, 'g')

                max_even = np.max(even_HE)
                if max_even > peak_height:
                    peak_height = max_even
                max_odd = np.max(odd_HE)
                if max_odd > peak_height:
                    peak_height = max_odd

#        plt.annotate(sname, (points[-1]*5.0E-9+t0[-1], height))
        plt.annotate(sname, (sname_X_loc, height),
                     textcoords=transform,
                     xycoords=transform)
        height += 2 * peak_height

    plt.show()
コード例 #27
0
    def setup(self):
        if self.ref_station in self.guess_timings:
            ref_T = self.guess_timings[self.ref_station]
            self.guess_timings = {
                station: T - ref_T
                for station, T in self.guess_timings.items()
                if station != self.ref_station
            }

        processed_data_folder = processed_data_dir(self.timeID)

        data_dir = processed_data_folder + "/" + self.output_folder
        if not isdir(data_dir):
            mkdir(data_dir)

        #Setup logger
        logging_folder = data_dir + '/logs_and_plots'
        if not isdir(logging_folder):
            mkdir(logging_folder)

        self.log = logger()
        self.log.set(logging_folder + "/log_out.txt"
                     )  ## TODo: save all output to a specific output folder
        self.log.take_stderr()
        self.log.take_stdout()

        print("timeID:", self.timeID)
        print("date and time run:", time.strftime("%c"))
        print("input folders:", self.pulse_input_folders)
        print("source IDs to fit:", self.sources_to_fit)
        print("guess locations:", self.guess_source_locations)
        print("polarization to use:", self.source_polarizations)
        print("source stations to exclude:", self.source_stations_to_exclude)
        print("source antennas to exclude:", self.source_antennas_to_exclude)
        print("bad antennas:", self.bad_ants)
        print("referance station:", self.ref_station)
        print("guess delays:", self.guess_timings)
        print()
        print()

        #### open data and data processing stuff ####
        print("loading data")

        ## needed for ant locs
        raw_fpaths = filePaths_by_stationName(self.timeID)
        raw_data_files = {
            sname: MultiFile_Dal1(fpaths, force_metadata_ant_pos=True)
            for sname, fpaths in raw_fpaths.items()
            if sname in chain(self.guess_timings.keys(), [self.ref_station])
        }

        #### sort antennas and stations ####
        self.station_order = list(
            self.guess_timings.keys()) + [self.ref_station]
        self.sorted_antenna_names = []
        self.station_to_antenna_index_dict = {}
        self.ant_loc_dict = {}

        for sname in self.station_order:
            first_index = len(self.sorted_antenna_names)

            stat_data = raw_data_files[sname]
            ant_names = stat_data.get_antenna_names()
            stat_ant_locs = stat_data.get_LOFAR_centered_positions()

            self.sorted_antenna_names += ant_names

            for ant_name, ant_loc in zip(ant_names, stat_ant_locs):
                self.ant_loc_dict[ant_name] = ant_loc

            self.station_to_antenna_index_dict[sname] = (
                first_index, len(self.sorted_antenna_names))

        self.ant_locs = np.zeros((len(self.sorted_antenna_names), 3))
        for i, ant_name in enumerate(self.sorted_antenna_names):
            self.ant_locs[i] = self.ant_loc_dict[ant_name]

        self.station_locations = {
            sname: self.ant_locs[self.station_to_antenna_index_dict[sname][0]]
            for sname in self.station_order
        }
        self.station_to_antenna_index_list = [
            self.station_to_antenna_index_dict[sname]
            for sname in self.station_order
        ]

        #### sort the delays guess, and account for station locations ####
        self.current_delays_guess = np.array(
            [self.guess_timings[sname] for sname in self.station_order[:-1]])
        self.original_delays = np.array(self.current_delays_guess)
        self.ant_recalibrate_order = np.array([
            i for i, antname in enumerate(self.sorted_antenna_names)
            if antname in self.antennas_to_recalibrate.keys()
        ],
                                              dtype=np.int)
        self.ant_recalibrate_guess = np.array([
            self.antennas_to_recalibrate[self.sorted_antenna_names[i]]
            for i in self.ant_recalibrate_order
        ])

        self.station_indeces = np.empty(len(self.sorted_antenna_names),
                                        dtype=np.int)
        for station_index, index_range in enumerate(
                self.station_to_antenna_index_list
        ):  ## note this DOES include ref stat
            first, last = index_range
            self.station_indeces[first:last] = station_index

        #### now we open the pulses ####
        input_manager = Part1_input_manager(processed_data_folder,
                                            self.pulse_input_folders)

        self.current_sources = []
        for knownID in self.sources_to_fit:
            source_ID, input_name = input_manager.known_source(knownID)

            print("prep fitting:", source_ID)

            polarity = self.source_polarizations[source_ID]

            source_to_add = source_object(
                source_ID, input_name,
                self.source_stations_to_exclude[source_ID],
                self.source_antennas_to_exclude[source_ID])
            source_to_add.prep_for_fitting(polarity, self)

            #quit()

            self.current_sources.append(source_to_add)

        self.num_sources = len(self.current_sources)
        self.num_delays = len(self.original_delays)
        self.num_RecalAnts = len(self.ant_recalibrate_order)
        self.num_antennas = len(self.sorted_antenna_names)
        self.num_measurments = self.num_sources * self.num_antennas

        self.current_solution = np.zeros(self.num_delays + self.num_RecalAnts +
                                         4 * self.num_sources)
        self.current_solution[:self.num_delays] = self.current_delays_guess
        self.current_solution[self.num_delays:self.num_delays +
                              self.num_RecalAnts] = self.ant_recalibrate_guess

        self.fitter = delay_fitter(self.ant_locs, self.station_indeces,
                                   self.ant_recalibrate_order,
                                   self.num_sources, self.num_delays)

        initial_param_i = self.num_delays + self.num_RecalAnts
        self.num_DOF = -self.num_delays - self.num_RecalAnts - 4 * self.num_sources
        for i, PSE in enumerate(self.current_sources):
            self.current_solution[initial_param_i + 4 * i:initial_param_i + 4 *
                                  (i + 1)] = self.guess_source_locations[
                                      PSE.ID]
            self.num_DOF += PSE.num_data()
            self.fitter.set_event(PSE.pulse_times)
コード例 #28
0
def get_noise_std(timeID,
                  initial_block,
                  max_num_blocks,
                  max_double_zeros=100,
                  stations=None,
                  polarization_flips="polarization_flips.txt",
                  bad_antennas="bad_antennas.txt"):

    processed_data_folder = processed_data_dir(timeID)
    if isinstance(polarization_flips, str):
        polarization_flips = read_antenna_pol_flips(processed_data_folder +
                                                    '/' + polarization_flips)
    if isinstance(processed_data_folder, str):
        bad_antennas = read_bad_antennas(processed_data_folder + '/' +
                                         bad_antennas)

    half_window_percent = 0.1
    half_window_percent *= 1.1  ## just to be sure we are away from edge

    raw_fpaths = filePaths_by_stationName(timeID)
    if stations is None:
        stations = raw_fpaths.keys()

    out_dict = {}
    for sname in stations:
        print(sname)

        TBB_data = MultiFile_Dal1(raw_fpaths[sname],
                                  polarization_flips=polarization_flips,
                                  bad_antennas=bad_antennas)
        RFI_filter = window_and_filter(timeID=timeID, sname=sname)
        block_size = RFI_filter.blocksize
        edge_size = int(half_window_percent * block_size)

        antenna_names = TBB_data.get_antenna_names()
        measured_std = np.zeros(len(antenna_names))
        measured_std[:] = -1

        for block_i in range(initial_block, initial_block + max_num_blocks):
            for ant_i in range(len(antenna_names)):
                if measured_std[ant_i] > 0:
                    continue  ## we got a measurment, so skip it

                data = np.array(TBB_data.get_data(block_i * block_size,
                                                  block_size,
                                                  antenna_index=ant_i),
                                dtype=np.double)

                if num_double_zeros(data) > max_double_zeros:
                    continue  ## bad block, skip

                filtered_data = np.real(RFI_filter.filter(data))
                filtered_data = filtered_data[edge_size:
                                              -edge_size]  ##avoid the edge
                measured_std[ant_i] = np.std(filtered_data)

            if np.all(measured_std > 0):  ## we can break early
                break

        for ant_name, std in zip(antenna_names, measured_std):
            out_dict[ant_name] = std

        filter = measured_std > 0
        print("   ave std:", np.average(measured_std[filter]))
        print("   total ant:", len(filter), "good ant:", np.sum(filter))
        print()

    return out_dict
コード例 #29
0
def plot_all_stations(event_ID, 
               timeID, output_folder, pulse_input_folders, guess_timings, sources_to_fit, guess_source_locations,
               source_polarizations, source_stations_to_exclude, source_antennas_to_exclude, bad_ants,
               antennas_to_recalibrate={}, min_ant_amplitude=10, ref_station="CS002"):
    
    processed_data_folder = processed_data_dir( timeID )
    file_manager = input_manager( processed_data_folder, pulse_input_folders )
    throw, input_Fname = file_manager.known_source( event_ID )
    
    data_file = h5py.File(input_Fname, "r")
    
    fpaths = filePaths_by_stationName( timeID )
    
    source_XYZT = guess_source_locations[event_ID]
    source_polarization = source_polarizations[event_ID]
    antennas_to_exclude = bad_ants + source_antennas_to_exclude[event_ID]
    
    ref_station_delay = 0.0
    if ref_station in guess_timings:
        ref_station_delay = guess_timings[ref_station]
    
    current_offset = 0
    for sname in data_file.keys():
        if (sname not in guess_timings and sname !=ref_station) or (sname in source_stations_to_exclude[event_ID]):
            continue
            
        stat_group = data_file[ sname ]
        station_file = MultiFile_Dal1(fpaths[sname], force_metadata_ant_pos=True)
            
        stat_delay = 0.0
        if sname != ref_station:
            stat_delay = guess_timings[sname] - ref_station_delay
        
        even_HEs = []
        odd_HEs = []
        
        even_offet = []
        odd_offset = []
        
        even_peakT_offset = []
        odd_peakT_offset = []
        
        peak_amp = 0.0
        ## first we open the data
        for ant_name, ant_loc in zip(station_file.get_antenna_names(),station_file.get_LOFAR_centered_positions()):
            if ant_name not in stat_group:
                continue
            
            even_ant_name = ant_name
            odd_ant_name = even_antName_to_odd( ant_name )
            
            ant_dataset = stat_group[ant_name]
#            even_trace = np.array( ant_dataset[0] )
            even_HE = np.array( ant_dataset[1] )
#            odd_trace = np.array( ant_dataset[2] )
            odd_HE = np.array( ant_dataset[3] )
            
            even_amp = np.max(even_HE)
            odd_amp = np.max(odd_HE)
            
            
            travel_delay = np.sqrt( (ant_loc[0]-source_XYZT[0])**2 + (ant_loc[1]-source_XYZT[1])**2 + (ant_loc[2]-source_XYZT[2])**2 )/v_air
            total_correction = travel_delay + stat_delay
            
            even_time =  - total_correction - source_XYZT[3]
            odd_time =  - total_correction
            
            if even_ant_name in antennas_to_recalibrate:
                    
                even_time -= antennas_to_recalibrate[even_ant_name]
                
            if odd_ant_name in antennas_to_recalibrate:
                odd_time -= antennas_to_recalibrate[odd_ant_name]  
                
            if source_polarization==2 and len(source_XYZT)==5:
                odd_time -= source_XYZT[4]
            else:
                odd_time -= source_XYZT[3]
                
            even_peak_time = ant_dataset.attrs["PolE_peakTime"] + even_time
            odd_peak_time = ant_dataset.attrs["PolO_peakTime"] + odd_time
                
            even_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolE_timeOffset_CS']
            odd_time += ant_dataset.attrs['starting_index']*5.0E-9 - ant_dataset.attrs['PolO_timeOffset_CS']
            
            even_is_good = (even_amp>min_ant_amplitude) and not (even_ant_name in antennas_to_exclude) 
            odd_is_good  = (odd_amp>min_ant_amplitude) and not (odd_ant_name in antennas_to_exclude)
                
            if (not even_is_good) and (not odd_is_good):
                continue
            
            M = max(even_amp, odd_amp)
            if M>peak_amp:
                peak_amp = M
                
            if even_is_good and source_polarization!=1:
                even_HEs.append( even_HE )
            else:
                even_HEs.append( None )
                
            if odd_is_good and source_polarization!=0:
                odd_HEs.append( odd_HE )
            else:
                odd_HEs.append( None )
        
            even_offet.append( even_time )
            odd_offset.append( odd_time )

            even_peakT_offset.append( even_peak_time )
            odd_peakT_offset.append( odd_peak_time )
            
        earliest_T = np.inf
        for pair_i in range(len(even_HEs)):
            
            if even_HEs[pair_i] is not None:
                even_T = np.arange( len(even_HEs[pair_i]) )*5.0E-9 + even_offet[pair_i]
                
                if even_T[0] < earliest_T:
                    earliest_T = even_T[0]
                
                p = plt.plot(even_T, even_HEs[pair_i]/peak_amp + current_offset)
                color = p[0].get_color()
                
                plt.plot( (even_peakT_offset[pair_i],even_peakT_offset[pair_i]), (current_offset,current_offset+1), color=color )
                
            if odd_HEs[pair_i] is not None:
                odd_T = np.arange( len(odd_HEs[pair_i]) )*5.0E-9 + odd_offset[pair_i]
                
                if odd_T[0] < earliest_T:
                    earliest_T = odd_T[0]
                
                p = plt.plot(odd_T, odd_HEs[pair_i]/peak_amp + current_offset)
                color = p[0].get_color()
                
                plt.plot( (odd_peakT_offset[pair_i],odd_peakT_offset[pair_i]), (current_offset,current_offset+1), color=color )

        plt.annotate(sname, xy=(earliest_T,current_offset+0.5))
        current_offset += 1
        
    plt.axvline(x=0)
    plt.show()
コード例 #30
0
    def open_files(self, log_func=do_nothing, force_metadata_ant_pos=True):

        #### open data files, and find RFI ####
        raw_fpaths = filePaths_by_stationName(self.timeID)
        self.station_names = []
        self.input_files = []
        self.RFI_filters = []
        self.station_to_antenna_indeces_dict = {}
        for station, fpaths in raw_fpaths.items():
            if (station not in self.stations_to_exclude) and (
                    self.use_core_stations_S1 or self.use_core_stations_S2 or
                (not station[:2] == 'CS') or station == "CS002"):
                log_func("opening", station)
                self.station_names.append(station)
                self.station_to_antenna_indeces_dict[station] = []

                new_file = MultiFile_Dal1(
                    fpaths, force_metadata_ant_pos=force_metadata_ant_pos)
                new_file.set_polarization_flips(self.pol_flips)
                self.input_files.append(new_file)

                RFI_result = None
                if self.do_RFI_filtering and self.use_saved_RFI_info:
                    self.RFI_filters.append(
                        window_and_filter(timeID=self.timeID, sname=station))

                elif self.do_RFI_filtering:
                    RFI_result = FindRFI(new_file,
                                         self.block_size,
                                         self.initial_RFI_block,
                                         self.RFI_num_blocks,
                                         self.RFI_max_blocks,
                                         verbose=False,
                                         figure_location=None)
                    self.RFI_filters.append(
                        window_and_filter(find_RFI=RFI_result))

                else:  ## only basic filtering
                    self.RFI_filters.append(
                        window_and_filter(blocksize=self.block_size))

        #### get antenna locations and delays ####

        self.num_antennas = len(self.interferometry_header.antenna_data)

        self.antenna_locations = np.zeros((self.num_antennas, 3),
                                          dtype=np.double)
        self.antenna_delays = np.zeros(self.num_antennas, dtype=np.double)
        self.antenna_data_offsets = np.zeros(self.num_antennas, dtype=np.long)
        self.half_antenna_data_length = np.zeros(self.num_antennas,
                                                 dtype=np.long)
        self.is_not_core = np.zeros(self.num_antennas, dtype=np.bool)
        self.antennas_to_use = []

        for ant_i, antenna_info in enumerate(
                self.interferometry_header.antenna_data):
            self.station_to_antenna_indeces_dict[antenna_info.station].append(
                ant_i)

            self.antenna_locations[ant_i] = antenna_info.location
            self.antenna_delays[ant_i] = antenna_info.timing_delay
            self.antenna_data_offsets[ant_i] = antenna_info.data_offset
            self.half_antenna_data_length[
                ant_i] = antenna_info.half_window_length

            self.is_not_core[ant_i] = (not antenna_info.station[:2] == 'CS'
                                       ) or antenna_info.station == "CS002"

            self.antennas_to_use.append(
                (self.station_names.index(antenna_info.station),
                 antenna_info.station_antenna_i))

        self.prefered_station_timing_offset = -0.000433356842721  #self.interferometry_header.h5_group.attrs["prefered_station_timing_offset"]
        pref_stat_i = self.station_names.index(self.prefered_station)
        self.prefered_station_antenna_timing_offsets = self.input_files[
            pref_stat_i].get_timing_callibration_delays()

        #### find prefered station info ####
        boundingBox_center = np.average(self.bounding_box, axis=-1)
        ant_i = self.station_to_antenna_indeces_dict[self.prefered_station][0]

        prefered_stat_shortestWindowTime = 0
        for ant_j in range(self.num_antennas):
            window_time, throw, throw = find_max_duration(
                self.antenna_locations[ant_i],
                self.antenna_locations[ant_j],
                self.bounding_box,
                center=boundingBox_center)
            if window_time > prefered_stat_shortestWindowTime:
                prefered_stat_shortestWindowTime = window_time

        log_func("prefered station:", self.prefered_station)

        self.startBlock_exclusion = int(
            0.1 * self.block_size)  ### TODO: save these to header.
        self.endBlock_exclusion = int(
            prefered_stat_shortestWindowTime / 5.0E-9) + 1 + int(
                0.1 * self.block_size)  ## last bit accounts for hann-window
        self.active_block_length = self.block_size - self.startBlock_exclusion - self.endBlock_exclusion  ##the length of block used for looking at data

        if (not self.use_core_stations_S1) or (not self.use_core_stations_S2):
            core_filtered_ant_locs = np.array(
                self.antenna_locations[self.is_not_core])
            core_filtered_ant_delays = np.array(
                self.antenna_delays[self.is_not_core])

        #### allocate some memory ####
        self.data_block = np.empty((self.num_antennas, self.block_size),
                                   dtype=np.complex)
        self.hilbert_envelope_tmp = np.empty(self.block_size, dtype=np.double)

        #### initialize stage 1 ####
        if self.use_core_stations_S1:
            self.trace_length_stage1 = 2 * np.max(
                self.half_antenna_data_length)
            S1_ant_locs = self.antenna_locations
            S1_ant_delays = self.antenna_delays
        else:
            self.trace_length_stage1 = 2 * np.max(
                self.half_antenna_data_length[self.is_not_core])
            S1_ant_locs = core_filtered_ant_locs
            S1_ant_delays = core_filtered_ant_delays
        self.trace_length_stage1 = 2**(int(np.log2(self.trace_length_stage1)) +
                                       1)
        self.stage_1_imager = II_tools.image_data_stage1(
            S1_ant_locs, S1_ant_delays, self.trace_length_stage1,
            self.upsample_factor)

        #### initialize stage 2 ####
        if self.use_core_stations_S2:
            S2_ant_locs = self.antenna_locations
            S2_ant_delays = self.antenna_delays
        else:
            S2_ant_locs = core_filtered_ant_locs
            S2_ant_delays = core_filtered_ant_delays
        self.trace_length_stage2 = 2**(int(np.log2(self.pulse_length)) + 1)
        self.stage_2_window = half_hann_window(self.pulse_length,
                                               self.hann_window_fraction)
        self.stage_2_imager = II_tools.image_data_stage2_absBefore(
            S2_ant_locs, S2_ant_delays, self.trace_length_stage2,
            self.upsample_factor)

        self.erasure_window = 1.0 - self.stage_2_window