def write_single_antenna_to_binary_file(input_file, dp_stand_id, polarization, output_file): """Extract a single dp_stand/pol to a npy file. Parameters ---------- input_file : string raw LWA-SV file path dp_stand_id : int stand id from 1 to 256 inclusive polarization : int antenna polarization output_file : string filename to be saved/appended to """ if not output_file.endswith(".singleAnt"): output_file = output_file + ".singleAnt" input_data = LWASVDataFile(input_file) with open(output_file, 'ab') as f: while input_data.get_remaining_frame_count() > 0: current_frame = input_data.read_frame() if current_frame.id == (dp_stand_id, polarization): float_arr = np.array(current_frame.data.iq).view(float) float_arr.tofile(f)
def extract_single_ant_from_middle(input_file, dp_stand_id, polarization, max_length=-1, tstart=0): """Extract and combine all data from a single antenna into a numpy array. Parameters ---------- input_file : string raw LWA-SV file path DP_stand_id : int stand id from 1 to 256 inclusive polarization : int antenna polarization max_length : int length in samples to extract tstart : int UTC timestamp (s since epoch) Returns ------- numpy array array of size (avail frames, bandwidth) """ input_data = LWASVDataFile(input_file) output_data = [] total_frames = input_data.get_remaining_frame_count() num_ants = input_data.getInfo()['nantenna'] samps_per_frame = 512 max_possible_length = math.ceil(total_frames / num_ants) * samps_per_frame if max_length < 0: max_length = max_possible_length print("-| {} frames in file".format(total_frames)) print("-| {} antennas in file".format(num_ants)) print("-| {} samples per frame".format(samps_per_frame)) print("--| Extracting from stand {}, pol {}".format( dp_stand_id, polarization)) print("--| Extracting {} of a possible {} samples".format( max_length, max_possible_length)) while len(output_data) < max_length: # while input_data.get_remaining_frame_count() > 0: current_frame = input_data.read_frame() current_tstamp = current_frame.getTime() if current_tstamp >= tstart: if current_frame.id == (dp_stand_id, polarization): for i in range(len(current_frame.data.iq)): output_data.append(current_frame.data.iq[i]) output_data = np.array(output_data) return output_data
def count_frames(filename): """Prints out the number of frames for each antenna from a TBN file Parameters ---------- filename : string name of file to be read (may end in dat, tbn, or nothing) """ def __getKeysByValue__(myDict, valueToFind): listOfKeys = [] listOfItems = myDict.items() for item in listOfItems: if item[1] == valueToFind: listOfKeys.append(item[0]) return listOfKeys bigDict = {} idfN = LWASVDataFile(filename) total_num_frames = idfN.get_remaining_frame_count() while idfN.get_remaining_frame_count() > 0: current_frame = idfN.read_frame() key = str(current_frame.id) try: bigDict[key] = bigDict[key] + 1 except KeyError: bigDict[key] = 1 # Make a list of unique frame counts unique_frame_counts = set(bigDict.values()) # Create dict with key = num_ants that each have value = num_frames antsFramesDict = {} for i in unique_frame_counts: num_frames = i num_ants = len(__getKeysByValue__(bigDict, num_frames)) antsFramesDict[num_ants] = num_frames total_calculated_frames = 0 print("STATS") print("-> Total number of frames in file: %s" % total_num_frames) for key, value in antsFramesDict.iteritems(): print("---> Number of antennas with %s frames: %s" % (value, key)) total_calculated_frames = total_calculated_frames + (key * value) print("SANITY CHECK") print("-> Frames") print("---> Sum of frames = {}".format(total_calculated_frames)) print("-> Antennas") print("---> Sum of antennas = {}".format(sum(antsFramesDict.keys())))
def make_sample_tbn(filename, num_frames=2000000, offset=0): """Takes the defined number of frames and writes them to a new .tbn file Parameters ---------- filename : string name of file to be read (may end in dat, tbn, or nothing) num_frames : int number of frames to be kept (default: 2000000) offset : float number of seconds to skip into file before reading (approximate) """ in_name = os.path.realpath(filename).split('/')[-1] in_base_name = in_name.split('.')[0] out_name = in_base_name + '.tbn' print(f"Reading data from {filename}") print(f"Writing data to {out_name}") if os.path.exists(out_name): raise RuntimeError( f"Output file {out_name} already exists - not going to overwrite it" ) in_tbn = LWASVDataFile(filename) out_fh = open(out_name, 'wb') if offset > 0: t = in_tbn.offset(offset) print(f"Requested offset: {offset} seconds") print(f"Achieved offset: {t} seconds") out_fh.write(in_tbn.fh.read(tbn.FRAME_SIZE * num_frames)) out_fh.close() in_tbn.close() print("\n{} TBN Size: {} kB".format(out_name, os.path.getsize(out_name) / 1024.)) # Check that the datatype is correct according to lsl out_tbn = LWASVDataFile(out_name) print("{} is of type: {} \n".format(out_name, type(out_tbn))) out_tbn.close()
def meta_to_txt(filename, outdir='./', station='lwasv'): """Pulls metadata from TBN file and puts it into a txt file of the same name Parameters ---------- filename : string name of file to be read (may end in dat, tbn, or nothing) outdir : string name of output directory to save textfile in station : string one of 'lwasv' or 'lwa1' for now """ simple_name = filename.split('/')[-1] simple_name = simple_name.split('.')[0] if not outdir.endswith('/'): outdir = outdir + '/' if not pathlib.Path(outdir).exists(): os.mkdir(outdir) print("{} TBN Size: {} kB".format(filename, os.path.getsize(filename) / 1024)) if station == 'lwasv': idfN = LWASVDataFile(filename) simple_name = simple_name + '-LWASV' elif station == 'lwa1': idfN = LWA1DataFile(filename) simple_name = simple_name + '-LWA1' else: raise NotImplementedError( "I haven't implemented that type of station yet") print("{} is of type: {}".format(filename, type(idfN))) # Poll the TBN file for its specifics with open(outdir + simple_name + ".txt", 'w') as meta: meta.write('TBN Metadata:\n') for key, value in idfN.get_info().items(): meta.write(" %s: %s\n" % (str(key), str(value))) idfN.close()
def pull_meta(filename, key): """ Pulls out metadata from a TBN file given a key. Possible keys: 'size','nframe','frame_size', 'nantenna','sample_rate','data_bits','start_time','start_time_samples','freq1' Parameters ---------- filename : string name of the file to pull the meta from key : string one of the possible keys listed above """ idfN = LWASVDataFile(filename) if key == 'Human start time': tbnKey = 'start_time' else: tbnKey = key value = idfN.get_info()[tbnKey] if key == 'Human start time': return str(value.utc_datetime) else: return str(value)
print(m_range) for x, l in enumerate(l_range): for y, m in enumerate(m_range): cost[x,y] = ls_cost([l, m], u, v, vis) plt.contourf(l_range, m_range, cost) plt.colorbar() plt.xlabel("l") plt.ylabel("m") plt.show() if __name__ == "__main__": plt.close('all') ants, n_baselines = select_antennas(station.antennas, use_pol=0) dfile = LWASVDataFile(tbn_filename) #baselines, visibilities = compute_visibilities(dfile, ants, target_freq) dfile.close() azimuth = station.get_pointing_and_distance(transmitter_coords + [0])[0] bl1d = project_baselines(baselines, azimuth) phases = np.angle(visibilities[0]) vis = visibilities[0] bl2d = np.array([np.array([b[0].stand.x - b[1].stand.x, b[0].stand.y - b[1].stand.y]) for b in baselines]) u = bl2d[:, 0] v = bl2d[:, 1]
def TBN_to_freq_bin_matrix_indexed_by_dp_stand(filename, Fc, f1, fft_size=512, Fs=100000, polarization=0): """Reads each from of a TBN, takes an FFT, and puts a single bin of it into an index particular to it's DP stand number. It continues to append bin values as so each index is the full time-series of frequency bin values of that DP stand. It concats the vectors to be the length of the shortest so that the resulting matrix is rectangular. *LIMITATION* : It only does one polarization. Parameters ---------- filename : string name of file to be read (may end in dat, tbn, or nothing) Fc : float center frequency in Hz f1 : float frequency of the signal to extract fft_size : int size of FFT window Fs : int sampling rate polarization : int which polarization to process, either 0 (default) or 1 Returns ------- numpy array array of size (num_dp_stands, samples_in_time_series) """ bin_of_f1 = get_frequency_bin(fc=Fc, f1=f1, fft_size=fft_size) input_data = LWASVDataFile(filename) lwasv = stations.lwasv num_stands = len(lwasv.stands) num_ants = num_stands / 2 #how many frames in total frame_count = input_data.get_remaining_frame_count() num_frames_per_ant = frame_count / num_ants # plus 1 to have space for a counter output_data = np.zeros((num_stands, num_frames_per_ant + 1), dtype=np.complex64) current_frame = input_data.read_frame() # iq_size = len(current_frame.data.iq) count = 1 while input_data.get_remaining_frame_count() > 0: (dp_stand_id, ant_polarization) = current_frame.id if ant_polarization == polarization: #NOT the same thing as the LWA stand number index = dp_stand_id - 1 # Which cell to write to count = int(np.real(output_data[index, 0]) + 1) if count < num_frames_per_ant: fft = np.fft.fftshift(np.fft.fft(current_frame.data.iq)) pt = fft[bin_of_f1] output_data[index, count] = pt # update counter output_data[index, 0] = count # Get frame for next iteration current_frame = input_data.read_frame() # Remove counter output_data = output_data[:, 1:] return output_data
def main(args): station = stations.lwasv tx_coords = known_transmitters.parse_args(args) print("Opening TBN file ({})".format(args.tbn_filename)) with LWASVDataFile(args.tbn_filename, ignore_timetag_errors=True) as tbnf: antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) if not args.hdf5_file: raise RuntimeError('Please provide an output filename') else: with build_output_file(h5_fname=args.hdf5_file, tbnf=tbnf, valid_ants=valid_ants, n_baselines=n_baselines, tx_freq=args.tx_freq, fft_len=args.fft_len, use_pfb=args.use_pfb, use_pol=args.use_pol, integration_length=args.integration_length, transmitter_coords=tx_coords) as h5f: if args.point_finding_alg == 'all' or args.point_finding_alg == 'peak': h5f.create_dataset_like('l_peak', h5f['l_est']) h5f.create_dataset_like('m_peak', h5f['m_est']) h5f.create_dataset_like('elevation_peak', h5f['elevation']) h5f.create_dataset_like('azimuth_peak', h5f['azimuth']) if args.point_finding_alg == 'all' or args.point_finding_alg == 'CoM': h5f.create_dataset_like('l_CoM', h5f['l_est']) h5f.create_dataset_like('m_CoM', h5f['m_est']) h5f.create_dataset_like('elevation_CoM', h5f['elevation']) h5f.create_dataset_like('azimuth_CoM', h5f['azimuth']) else: raise NotImplementedError( f"Unrecognized point finding algorithm: {args.point_finding_alg}" ) del h5f['l_est'] del h5f['m_est'] del h5f['elevation'] del h5f['azimuth'] k = 0 save_all_sky = (args.all_sky and k in args.all_sky) or ( args.all_sky_every and k % args.all_sky_every == 0 ) # or (args.scatter_bad_fits and skip) if save_all_sky: fig, ax = plt.subplots() for bl, freqs, vis in compute_visibilities_gen( tbnf, valid_ants, integration_length=args.integration_length, fft_length=args.fft_len, use_pol=args.use_pol, use_pfb=args.use_pfb): gridded_image = grid_visibilities(bl, freqs, vis, args.tx_freq, station) save_all_sky = (args.all_sky and k in args.all_sky) or ( args.all_sky_every and k % args.all_sky_every == 0) if args.point_finding_alg == 'all' or 'peak': result = get_gimg_max(gridded_image, return_img=save_all_sky) l = result[0] m = result[1] src_elev, src_az = lm_to_ea(l, m) h5f['l_peak'][k] = l h5f['m_peak'][k] = m h5f['elevation_peak'][k] = src_elev h5f['azimuth_peak'][k] = src_az if args.point_finding_alg == 'all' or args.point_finding_alg == 'CoM': result = get_gimg_center_of_mass( gridded_image, return_img=save_all_sky) l = result[0] m = result[1] src_elev, src_az = lm_to_ea(l, m) h5f['l_CoM'][k] = l h5f['m_CoM'][k] = m h5f['elevation_CoM'][k] = src_elev h5f['azimuth_CoM'][k] = src_az if save_all_sky: img = result[2] extent = result[3] ax.imshow(img, extent=extent, origin='lower', interpolation='nearest') plt.savefig('allsky_int_{}.png'.format(k)) k += 1 print("\n\n") if args.stop_after >= 0 and k >= args.stop_after: break
def generate_multiple_ants(input_file, dp_stand_ids, polarization, chunk_length=2**20, max_length=-1, truncate=True): """Generate chunks of data from a list of antennas. Parameters ---------- input_file : string raw LWA-SV file path dp_stand_ids : list list of stand ids from 1 to 256 inclusive polarization : list antenna polarization chunk_length : int length of each chunk to extract truncate : bool if the last chunk is shorter than chunk_length, return a short chunk otherwise, pad with zeros Returns ------- numpy array array of size (avail frames, bandwidth) """ input_data = LWASVDataFile(input_file) total_frames = input_data.get_remaining_frame_count() num_ants = input_data.get_info()['nantenna'] samps_per_frame = 512 max_possible_length = int( math.ceil(total_frames / num_ants) * samps_per_frame) if max_length < 0: max_length = max_possible_length print("-| {} frames in file".format(total_frames)) print("-| {} antennas in file".format(num_ants)) print("-| {} samples per frame".format(samps_per_frame)) print("--| Extracting from stands {}, pol {}".format( dp_stand_ids, polarization)) print("--| There are possibly {} samples for each stand".format( max_possible_length)) print("--| Returning data in chunks of length {}".format(chunk_length)) if chunk_length < samps_per_frame: raise ValueError( "--| Error: chunk size ({}) must be larger than frame size ({} samples)" .format(chunk_length, samps_per_frame)) # preallocate array to hold the current chunk of data. leave some space for overflow chunk_buffer = np.empty((len(dp_stand_ids), int(chunk_length * 2)), dtype=np.complex64) done = False samples_sent = 0 file_ended = False compensating_start_times = True dropped_frames = 0 start_times = [0] * len(dp_stand_ids) fill_levels = [0] * len(dp_stand_ids) while not done: # fill the chunk buffer while any([l < chunk_length for l in fill_levels]): # read a frame try: current_frame = input_data.read_frame() except errors.eofError: file_ended = True break current_id = current_frame.id for out_idx, stand in enumerate(dp_stand_ids): if (stand, polarization) == current_id: # this is the right stand, add to the buffer if compensating_start_times: time = current_frame.time if time >= max(start_times): start_times[out_idx] = time chunk_buffer[ out_idx][: samps_per_frame] = current_frame.data.iq fill_levels[out_idx] = samps_per_frame if start_times.count(start_times[0]) == len( start_times) and start_times[0] > 0: compensating_start_times = False print("--| Start times match at time {:f}".format( time)) else: wr_idx = fill_levels[out_idx] if wr_idx + samps_per_frame > chunk_buffer.shape[1]: extend_by = max(int(0.2 * chunk_buffer.shape[1]), samps_per_frame) extension = np.empty( (chunk_buffer.shape[0], extend_by)) print( "--|Chunk buffer overflowed, increasing length from {} to {}" .format(chunk_buffer.shape[1], chunk_buffer.shape[1] + extend_by)) chunk_buffer = np.concatenate( (chunk_buffer, extension), axis=1) chunk_buffer[ out_idx][wr_idx:wr_idx + samps_per_frame] = current_frame.data.iq fill_levels[out_idx] += samps_per_frame break if samples_sent + chunk_length >= max_length: # this is the last chunk print("--| Requested number of samples read") done = True last_chunk_len = max_length - samples_sent if not truncate: chunk_buffer[:, last_chunk_len:] = 0 yield chunk_buffer[:, :last_chunk_len] elif file_ended: # return unfinished chunk print("--| Reached end of file") min_fill = min(fill_levels) done = True if not truncate: chunk_buffer[:, last_chunk_len:] = 0 yield chunk_buffer[:, :min_fill] else: # yield the chunk yield chunk_buffer[:, :chunk_length] samples_sent += chunk_length for i in range(len(chunk_buffer)): # check if there's more than a chunk of samples for any of the stands if fill_levels[i] > chunk_length: # copy the extra samples to the start of the buffer overflow_length = fill_levels[i] - chunk_length chunk_buffer[i][0:overflow_length] = chunk_buffer[i][ chunk_length:fill_levels[i]] # start the next read after the extra samples fill_levels[i] = overflow_length else: # otherwise we can overwrite the whole buffer fill_levels[i] = 0 return
def extract_single_ant(input_file, dp_stand_id, polarization, max_length=-1, file_is_lwasvdatafile=False, fill_missing=False): """Extract and combine all data from a single antenna into a numpy array. Parameters ---------- input_file : string raw LWA-SV file path DP_stand_id : int stand id from 1 to 256 inclusive polarization : int antenna polarization max_length : int length in samples to extract Returns ------- numpy array array of size (avail frames, bandwidth) """ if file_is_lwasvdatafile: input_data = input_file else: input_data = LWASVDataFile(input_file) output_data = [] total_frames = input_data.get_remaining_frame_count() num_ants = input_data.get_info()['nantenna'] samps_per_frame = 512 max_possible_length = math.ceil(total_frames / num_ants) * samps_per_frame expected_timetag_delta = int( samps_per_frame / input_data.get_info()['sample_rate'] * reference_fs) prev_timetag = None if max_length < 0: max_length = max_possible_length print("-| {} frames in file".format(total_frames)) print("-| {} antennas in file".format(num_ants)) print("-| {} samples per frame".format(samps_per_frame)) print("--| Extracting from stand {}, pol {}".format( dp_stand_id, polarization)) print("--| Extracting {} of a possible {} samples".format( max_length, max_possible_length)) print("--| Expecting a timetag skip of {} between frames".format( expected_timetag_delta)) # while input_data.get_remaining_frame_count() > 0: while len(output_data) < max_length: try: current_frame = input_data.read_frame() except errors.EOFError: break if current_frame.id != (dp_stand_id, polarization): continue if prev_timetag is not None: if current_frame.payload.timetag != prev_timetag + expected_timetag_delta: warnings.warn( f"WARNING: invalid timetag skip in sample {len(output_data) + 1}" ) print( f"Expected {expected_timetag_delta + prev_timetag} but got {current_frame.payload.timetag}" ) print( f"This is a difference of {current_frame.payload.timetag - prev_timetag} as opposed to the expected difference of {expected_timetag_delta}" ) if fill_missing: timetag_diff = current_frame.payload.timetag - prev_timetag frames_dropped = int(timetag_diff / expected_timetag_delta) print( f"Filling {frames_dropped} missing frames with zeros ({samps_per_frame * frames_dropped} samples)" ) print(len(output_data)) for i in range(samps_per_frame * frames_dropped): output_data.append(0.0) print(len(output_data)) for i in range(len(current_frame.payload.data)): output_data.append(current_frame.payload.data[i]) prev_timetag = current_frame.payload.timetag output_data = np.array(output_data) return output_data
def extract_multiple_ants(input_file, dp_stand_ids, polarization, max_length=-1, truncate=True): """Extract and combine all data from a list of antenna into an array of numpy arrays. Parameters ---------- input_file : string raw LWA-SV file path dp_stand_ids : list list of stand ids from 1 to 256 inclusive polarization : list antenna polarization max_length : int length in samples to extract truncate : boolean discard later frames so all antennas have the same number Returns ------- numpy array array of size (len(dp_stand_ids), avail frames) """ input_data = LWASVDataFile(input_file) total_frames = input_data.get_remaining_frame_count() num_ants = input_data.get_info()['nantenna'] samps_per_frame = 512 max_possible_length = int( math.ceil(total_frames / num_ants) * samps_per_frame) if max_length < 0: max_length = max_possible_length print("-| {} frames in file".format(total_frames)) print("-| {} antennas in file".format(num_ants)) print("-| {} samples per frame".format(samps_per_frame)) print("--| Extracting from stands {}, pol {}".format( dp_stand_ids, polarization)) print( "--| Attempting to extract {} of a possible {} samples for each stand". format(max_length, max_possible_length)) # preallocate data array a little bigger than we think the longest signal will be output_data = np.zeros((len(dp_stand_ids), int(max_length * 1.2) + 1), dtype=np.complex64) fill_levels = [0] * len(dp_stand_ids) # while input_data.get_remaining_frame_count() > 0: while any([l < max_length for l in fill_levels]): try: current_frame = input_data.read_frame() except errors.eofError: print("--| EOF reached before maximum length.") break current_id = current_frame.id # check if this frame is one we want matching_stand = next( (s for s in dp_stand_ids if (s, polarization) == current_id), -1) for s in dp_stand_ids: if (s, polarization) == current_id: out_index = dp_stand_ids.index(matching_stand) if fill_levels[out_index] < max_length: wr_idx = fill_levels[out_index] output_data[ out_index][wr_idx:wr_idx + samps_per_frame] = current_frame.data.iq fill_levels[out_index] += samps_per_frame break min_fill = min(fill_levels) # if the lengths are unequal then truncate long ones if truncate and fill_levels.count(min_fill) != len(fill_levels): print("--| Truncating lengths from {} to {}".format( fill_levels, min_fill)) return output_data[:, :min_fill] else: return output_data[:, :max_length]
def main(args): # saz and sel are used later img = aipy.img.ImgW(size=50, res=0.5) top = img.get_top(center=(50, 50)) saz, sel = aipy.coord.top2azalt(top) station = stations.lwasv tx_coords = known_transmitters.parse_args(args) print("Opening TBN file ({})".format(args.tbn_filename)) with LWASVDataFile(args.tbn_filename, ignore_timetag_errors=True) as tbnf: antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) if not args.hdf5_file: raise RuntimeError('Please provide an output filename') else: with build_output_file( h5_fname=args.hdf5_file, tbnf=tbnf, valid_ants=valid_ants, n_baselines=n_baselines, # use_pfb=args.use_pfb, use_pol=args.use_pol, integration_length=args.integration_length, transmitter_coords=tx_coords) as h5f: del h5f['l_est'] del h5f['m_est'] freq = tbnf.get_info('freq1') idx = [ant.digitizer - 1 for ant in valid_ants] xyz = np.array([[ant.stand.x, ant.stand.y, ant.stand.z] for ant in valid_ants]) delays = np.array( [ant.cable.delay(freq) for ant in valid_ants]) delays -= delays.min() n_samples = tbnf.get_info('nframe') / tbnf.get_info('nantenna') samples_per_integration = int(args.integration_length * tbnf.get_info('sample_rate') / 512) n_integrations = int( np.floor(n_samples / samples_per_integration)) for int_num in range(n_integrations): print( f"Starting iteration {int_num + 1} of {n_integrations}" ) # Load in the data and select what we need tInt, t0, data = tbnf.read(args.integration_length) data = data[idx, :] # Apply a phase rotation to deal with the cable delays for i in range(data.shape[0]): data[i, :] *= np.exp(2j * np.pi * freq * delays[i]) data /= (np.abs(data)).max() # Calculate Rx - The time-averaged autocorrelation matrix nSamp = data.shape[1] xOutput = [] print("Computing time-averaged autocorrelation matrix") for i in range(nSamp): x = np.matrix(data[:, i]).T xOutput.append(x) try: Rx += x * x.H except: Rx = x * x.H Rx /= nSamp # Find the eigenvectors/values for Rx and order them by significance print("Computing eigenvectors/values of the ACM") w, v = np.linalg.eig(Rx) order = np.argsort(np.abs(w))[::-1] w = w[order] v = v[:, order] # Break the eigenvalues into a signal sub-space, Us, and a noise sub- # space, Un. This is currently done based on the number of sources # we have rather than inferred from the eigenvalues. ##Us = numpy.where( numpy.abs(w) > sigma )[0] #TODO I think this part should help find frequency too but Jayce had it commented out because the sigma section wasn't working (see her tbnMusic.py script I think) ##Un = numpy.where( numpy.abs(w) <= sigma )[0] # Us = range(3) #TODO What Jayce had. I imagine she had 4 sources # Un = range(3, w.size) Us = range(1) Un = range(1, w.size) print("Evaluating MUSIC spectrum") P = np.zeros_like(saz) E = np.zeros_like(saz) for i in range(saz.shape[0]): print( f"Starting row {i+1} / {saz.shape[0]} for integration {int_num}" ) for j in range(saz.shape[1]): ta = saz[i, j] te = sel[i, j] if not np.isfinite(ta) or not np.isfinite(te): continue pv = np.array([ np.cos(te) * np.sin(ta), np.cos(te) * np.cos(ta), np.sin(te) ]) a = np.zeros((len(valid_ants), 1), dtype=np.complex128) for k in range(len(valid_ants)): a[k, 0] = np.exp( 2j * np.pi * freq * np.dot(xyz[k, :] - xyz[0, :], pv) / speedOfLight) a = np.matrix(a) v2 = np.matrix(v[:, Un]) o = a.H * v2 * v2.H * a P[i, j] = 1.0 / max([1e-9, o[0, 0].real]) spectrum_max_idx = np.where(P == P.max()) el_max = sel[spectrum_max_idx][0] az_max = saz[spectrum_max_idx][0] h5f['elevation'][int_num] = el_max h5f['azimuth'][int_num] = az_max print( f"Integration complete - az = {az_max:.2f} el = {el_max:.2f}" )
def main(args): # this first part of the code is run by all processes # set up MPI environment comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if size < 2: raise RuntimeError( f"This program requires at least two MPI processes to function. Please rerun with more resources" ) # designate the last process as the supervisor/file reader supervisor = size - 1 # open the TBN file for reading tbnf = LWASVDataFile(args.tbn_filename, ignore_timetag_errors=True) # figure out the details of the run we want to do tx_coords = known_transmitters.parse_args(args) antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) n_ants = len(valid_ants) sample_rate = tbnf.get_info('sample_rate') # some of our TBNs claim to have frame size 1024 but they are lying frame_size = 512 tbn_center_freq = tbnf.get_info('freq1') total_integrations, _ = compute_integration_numbers( tbnf, args.integration_length) # open the output HDF5 file and create datasets # because of the way parallelism in h5py works all processes (even ones # that don't write to the file) must do this h5f = build_output_file(args.hdf5_file, tbnf, valid_ants, n_baselines, args.integration_length, tx_freq=args.tx_freq, fft_len=args.fft_len, use_pfb=args.use_pfb, use_pol=args.use_pol, opt_method=opt_method, vis_model='gaussian', transmitter_coords=tx_coords, mpi_comm=comm) if rank == supervisor: # the supervisor process runs this code print("supervisor: started") # state info reached_end = False workers_alive = [True for _ in range(size - 1)] int_no = 0 while True: if not reached_end: # grab data for the next available worker try: duration, start_time, data = tbnf.read( args.integration_length) # only use data from valid antennas data = data[[a.digitizer - 1 for a in valid_ants], :] except EOFError: reached_end = True print(f"supervisor: reached EOF") if int_no >= total_integrations: print(f"supervisor: this is the last integration") reached_end = True # get the next "ready" message from the workers st = MPI.Status() msg = comm.recv(status=st) if msg == "ready": print( f"supervisor: received 'ready' message from worker {st.source}" ) # if we're done, send an exit message and mark that we've killed this worker # an empty array indicates that the worker should exit if reached_end: print( f"supervisor: sending exit message to worker {st.source}" ) comm.Send(np.array([]), dest=st.source, tag=int_no) workers_alive[st.source] = False if not any(workers_alive): print(f"supervisor: all workers told to exit, goodbye") break # otherwise, send the data to the worker for processing else: print( f"supervisor: sending data for integration {int_no}/{total_integrations} to worker {st.source}" ) # Send with a capital S is optimized to send numpy arrays comm.Send(data, dest=st.source, tag=int_no) int_no += 1 else: raise ValueError( f"Supervisor received unrecognized message '{msg}' from worker {st.source}" ) tbnf.close() else: # the worker processes run this code print(f"worker {rank} started") # workers don't need access to the TBN file tbnf.close() # figure out the size of the incoming data buffer samples_per_integration = int( round(args.integration_length * sample_rate / frame_size)) * frame_size buffer_shape = (n_ants, samples_per_integration) while True: # send with a lowercase s can send any pickle-able python object # this is a synchronous send - it will block until the message is read by the supervisor # the other sends (e.g. comm.Send) only block until the message is safely taken by MPI, which might happen before the receiver actually reads it comm.ssend("ready", dest=supervisor) # build a buffer to be filled with data data = np.empty(buffer_shape, np.complex64) # receive the data from the supervisor st = MPI.Status() comm.Recv(data, source=supervisor, status=st) int_no = st.tag # if the buffer is empty, we're done if st.count == 0: print(f"worker {rank}: received exit message, exiting") break # otherwise process the data we've recieved print( f"worker {rank}: received data for integration {int_no}, starting processing" ) # run the correlator bl, freqs, vis = fxc.FXMaster( data, valid_ants, LFFT=args.fft_len, pfb=args.use_pfb, sample_rate=sample_rate, central_freq=tbn_center_freq, Pol='xx' if args.use_pol == 0 else 'yy', return_baselines=True, gain_correct=True) # extract the frequency bin we want target_bin = np.argmin([abs(args.tx_freq - f) for f in freqs]) vis_tbin = vis[:, target_bin] # baselines in wavelengths uvw = uvw_from_antenna_pairs(bl, wavelength=3e8 / args.tx_freq) # model fitting l_out, m_out, opt_result = fit_model_to_vis(uvw, vis_tbin, residual_function, l_init, m_init, verbose=False) # convert direction cosines to sky coords src_elev, src_az = lm_to_ea(l_out, m_out) # write data to h5 file h5f['l_start'][int_no] = l_init h5f['m_start'][int_no] = m_init h5f['l_est'][int_no] = l_out h5f['m_est'][int_no] = m_out h5f['elevation'][int_no] = src_elev h5f['azimuth'][int_no] = src_az h5f['cost'][int_no] = opt_result['cost'] h5f['nfev'][int_no] = opt_result['nfev'] # compute the bin power and save it to the file # arbitrarily picking the tenth antenna in this list power_calc_data = data[10, :] h5f['snr_est'][int_no] = estimate_snr(power_calc_data, args.fft_len, args.tx_freq, sample_rate, tbn_center_freq) print(f"worker {rank}: done processing integration {int_no}") # back to common code for both supervisor and workers h5f.attrs['total_integrations'] = int_no h5f.close()
def main(args): # this first part of the code is run by all processes # set up MPI environment comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if size < 2: raise RuntimeError( f"This program requires at least two MPI processes to function. Please rerun with more resources" ) # designate the last process as the supervisor/file reader supervisor = size - 1 # open the TBN file for reading tbnf = LWASVDataFile(args.tbn_filename, ignore_timetag_errors=True) # figure out the details of the run we want to do tx_coords = known_transmitters.parse_args(args) antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) n_ants = len(valid_ants) total_integrations, _ = compute_integration_numbers( tbnf, args.integration_length) sample_rate = tbnf.get_info('sample_rate') # some of our TBNs claim to have frame size 1024 but they are lying frame_size = 512 tbn_center_freq = tbnf.get_info('freq1') # open the output HDF5 file and create datasets # because of the way parallelism in h5py works all processes (even ones # that don't write to the file) must do this h5f = build_output_file(args.hdf5_file, tbnf, valid_ants, n_baselines, args.integration_length, tx_freq=args.tx_freq, fft_len=args.fft_len, use_pfb=args.use_pfb, use_pol=args.use_pol, transmitter_coords=tx_coords, mpi_comm=comm) if args.point_finding_alg == 'all' or args.point_finding_alg == 'peak': h5f.create_dataset_like('l_peak', h5f['l_est']) h5f.create_dataset_like('m_peak', h5f['m_est']) h5f.create_dataset_like('elevation_peak', h5f['elevation']) h5f.create_dataset_like('azimuth_peak', h5f['azimuth']) if args.point_finding_alg == 'all' or args.point_finding_alg == 'CoM': h5f.create_dataset_like('l_CoM', h5f['l_est']) h5f.create_dataset_like('m_CoM', h5f['m_est']) h5f.create_dataset_like('elevation_CoM', h5f['elevation']) h5f.create_dataset_like('azimuth_CoM', h5f['azimuth']) else: raise NotImplementedError( f"Unrecognized point finding algorithm: {args.point_finding_alg}") del h5f['l_est'] del h5f['m_est'] del h5f['elevation'] del h5f['azimuth'] if rank == supervisor: # the supervisor process runs this code print("supervisor: started") # state info reached_end = False workers_alive = [True for _ in range(size - 1)] int_no = 0 while True: if not reached_end: # grab data for the next available worker try: duration, start_time, data = tbnf.read( args.integration_length) # only use data from valid antennas data = data[[a.digitizer - 1 for a in valid_ants], :] except EOFError: reached_end = True print(f"supervisor: reached EOF") if int_no >= total_integrations: print(f"supervisor: this is the last integration") reached_end = True # get the next "ready" message from the workers st = MPI.Status() msg = comm.recv(status=st) if msg == "ready": print( f"supervisor: received 'ready' message from worker {st.source}" ) # if we're done, send an exit message and mark that we've killed this worker # an empty array indicates that the worker should exit if reached_end: print( f"supervisor: sending exit message to worker {st.source}" ) comm.Send(np.array([]), dest=st.source, tag=int_no) workers_alive[st.source] = False if not any(workers_alive): print(f"supervisor: all workers told to exit, goodbye") break # otherwise, send the data to the worker for processing else: print( f"supervisor: sending data for integration {int_no}/{total_integrations} to worker {st.source}" ) # Send with a capital S is optimized to send numpy arrays comm.Send(data, dest=st.source, tag=int_no) int_no += 1 else: raise ValueError( f"Supervisor received unrecognized message '{msg}' from worker {st.source}" ) tbnf.close() else: # the worker processes run this code print(f"worker {rank} started") # workers don't need access to the TBN file tbnf.close() # figure out the size of the incoming data buffer samples_per_integration = int( round(args.integration_length * sample_rate / frame_size)) * frame_size buffer_shape = (n_ants, samples_per_integration) while True: # send with a lowercase s can send any pickle-able python object # this is a synchronous send - it will block until the message is read by the supervisor # the other sends (e.g. comm.Send) only block until the message is safely taken by MPI, which might happen before the receiver actually reads it comm.ssend("ready", dest=supervisor) # build a buffer to be filled with data data = np.empty(buffer_shape, np.complex64) # receive the data from the supervisor st = MPI.Status() comm.Recv(data, source=supervisor, status=st) int_no = st.tag # if the buffer is empty, we're done if st.count == 0: print(f"worker {rank}: received exit message, exiting") break # otherwise process the data we've recieved print( f"worker {rank}: received data for integration {int_no}, starting processing" ) # run the correlator bl, freqs, vis = fxc.FXMaster( data, valid_ants, LFFT=args.fft_len, pfb=args.use_pfb, sample_rate=sample_rate, central_freq=tbn_center_freq, Pol='xx' if args.use_pol == 0 else 'yy', return_baselines=True, gain_correct=True) gridded_image = grid_visibilities(bl, freqs, vis, args.tx_freq, station) save_all_sky = (args.all_sky and int_no in args.all_sky) or ( args.all_sky_every and int_no % args.all_sky_every == 0) if args.point_finding_alg == 'all' or 'peak': result = get_gimg_max(gridded_image, return_img=save_all_sky) l = result[0] m = result[1] src_elev, src_az = lm_to_ea(l, m) h5f['l_peak'][int_no] = l h5f['m_peak'][int_no] = m h5f['elevation_peak'][int_no] = src_elev h5f['azimuth_peak'][int_no] = src_az if args.point_finding_alg == 'all' or args.point_finding_alg == 'CoM': result = get_gimg_center_of_mass(gridded_image, return_img=save_all_sky) l = result[0] m = result[1] src_elev, src_az = lm_to_ea(l, m) h5f['l_CoM'][int_no] = l h5f['m_CoM'][int_no] = m h5f['elevation_CoM'][int_no] = src_elev h5f['azimuth_CoM'][int_no] = src_az if save_all_sky: img = result[2] extent = result[3] fig, ax = plt.subplots() ax.imshow(img, extent=extent, origin='lower', interpolation='nearest') plt.savefig('allsky_int_{}.png'.format(int_no)) # compute the bin power and save it to the file # arbitrarily picking the tenth antenna in this list power_calc_data = data[10, :] h5f['snr_est'][int_no] = estimate_snr(power_calc_data, args.fft_len, args.tx_freq, sample_rate, tbn_center_freq) print(f"worker {rank}: done processing integration {int_no}") # back to common code for both supervisor and workers h5f.attrs['total_integrations'] = int_no h5f.close()
def main(args): start = time.time() print("\nCreating filenames and checking input file extension") input_file = args.filename ext = os.path.splitext(input_file)[-1].lower() if ext not in ['', '.tbn']: raise Exception("Extension should be .tbn or not exist") else: input_filename = os.path.split(os.path.splitext(input_file)[-2])[-1] output_file = input_filename + '.hdf5' print("-| Input file extension is {} (full name: {})".format(ext, input_file)) print("-| Output file extension is '.hdf5 (full name: {})".format(output_file)) print("\nChecking input data") input_data = LWASVDataFile(input_file) # For getting output array size lwasv = stations.lwasv num_stands = len(lwasv.stands) num_ants = num_stands/2 min_frames = get_min_frame_count(input_file) print("-| Minimum number of frames is: {}".format(min_frames)) print("-| Number of antennas per polarization: {}".format(num_ants)) # Annoying to do this here current_frame = input_data.read_frame() iq_size = len(current_frame.data.iq) # Shape is the datasize plus 1 for a counter at each element # output_shape_with_counter = (num_ants, min_frames * iq_size + 1) output_shape = (num_ants, min_frames * iq_size) pol0_counters = np.zeros(num_ants, dtype=int) pol1_counters = np.zeros(num_ants, dtype=int) print("-| Shape of each output dataset will be {}".format(output_shape)) print("\nCreating and opening output file") with h5py.File(output_file, "w") as f: # Create a group to store everything in, and to attach attributes to print("-| Creating parent group {}[{}]".format(output_file, input_filename)) parent = f.create_group(input_filename) # Add attributes to group print("-| Adding TBN metadata as attributes to the parent group") for key, value in input_data.get_info().items(): if key is "start_time": parent.attrs["Human start time"] = str(value.utc_datetime) parent.attrs[key] = value print("--| key: {} | value: {}".format(key, value)) # Create a subdataset for each polarization print("-| Creating datasets full of zeros") pol0 = parent.create_dataset("pol0", output_shape, dtype=np.complex64)#, compression='lzf') pol1 = parent.create_dataset("pol1", output_shape, dtype=np.complex64)#, compression='lzf') # For progress bar totalFrames = input_data.get_remaining_frame_count() current_iteration = 0 print("-| Beginning to build output from input") while input_data.get_remaining_frame_count() > 0: current_iteration += 1 printProgressBar(current_iteration, totalFrames) (frame_dp_stand_id, frame_ant_polarization) = current_frame.id frameData = current_frame.data.iq x_index = frame_dp_stand_id - 1 if frame_ant_polarization == 0: counter = pol0_counters dset = pol0 elif frame_ant_polarization == 1: counter = pol1_counters dset = pol1 y_index = counter[x_index] if not isFrameLimited(y_index, len(frameData), min_frames): data_start = y_index data_end = data_start+len(frameData) dset[x_index, data_start:data_end] = frameData counter[x_index] = data_end # Get frame for next iteration current_frame = input_data.read_frame() print("\nDONE") end = time.time() totalTime = end-start print("\nThis script ran for {}s = {}min = {}h".format(totalTime, totalTime/60, totalTime/3600))
def main(args): # Parse command line options filename = args.filename # Setup the LWA station information if args.metadata is not None: try: station = stations.parse_ssmif(args.metadata) except ValueError: station = metabundleADP.get_station(args.metadata, apply_sdm=True) else: station = stations.lwasv antennas = station.antennas idf = LWASVDataFile(filename) if not isinstance(idf, TBFFile): raise RuntimeError("File '%s' does not appear to be a valid TBF file" % os.path.basename(filename)) jd = idf.get_info('start_time').jd date = idf.get_info('start_time').datetime nFpO = idf.get_info('nchan') // 12 sample_rate = idf.get_info('sample_rate') nInts = idf.get_info('nframe') // nFpO # Get valid stands for both polarizations goodX = [] goodY = [] for i in range(len(antennas)): ant = antennas[i] if ant.combined_status != 33 and not args.all: pass else: if ant.pol == 0: goodX.append(ant) else: goodY.append(ant) # Now combine both lists to come up with stands that # are in both so we can form the cross-polarization # products if we need to good = [] for antX in goodX: for antY in goodY: if antX.stand.id == antY.stand.id: good.append(antX.digitizer - 1) good.append(antY.digitizer - 1) # Report on the valid stands found. This is a little verbose, # but nice to see. print("Found %i good stands to use" % (len(good) // 2, )) for i in good: print("%3i, %i" % (antennas[i].stand.id, antennas[i].pol)) # Number of frames to read in at once and average nFrames = min([int(args.avg_time * sample_rate), nInts]) args.offset = idf.offset(args.offset) nSets = idf.get_info('nframe') // nFpO // nFrames nSets = nSets - int(args.offset * sample_rate) // nFrames central_freq = idf.get_info('freq1') central_freq = central_freq[len(central_freq) // 2] print("Data type: %s" % type(idf)) print("Samples per observations: %i" % nFpO) print("Sampling rate: %i Hz" % sample_rate) print("Tuning frequency: %.3f Hz" % central_freq) print("Captures in file: %i (%.3f s)" % (nInts, nInts / sample_rate)) print("==") print("Station: %s" % station.name) print("Date observed: %s" % date) print("Julian day: %.5f" % jd) print("Offset: %.3f s (%i frames)" % (args.offset, args.offset * sample_rate)) print("Integration Time: %.3f s" % (nFrames / sample_rate)) print("Number of integrations in file: %i" % nSets) # Make sure we don't try to do too many sets if args.samples > nSets: args.samples = nSets # Loop over junks of 100 integrations to make sure that we don't overflow # the FITS IDI memory buffer s = 0 leftToDo = args.samples basename = os.path.split(filename)[1] basename, ext = os.path.splitext(basename) while leftToDo > 0: fitsFilename = "%s.FITS_%i" % ( basename, (s + 1), ) if leftToDo > 100: chunk = 100 else: chunk = leftToDo process_chunk(idf, station, good, fitsFilename, int_time=args.avg_time, pols=args.products, chunk_size=chunk) s += 1 leftToDo = leftToDo - chunk idf.close()
def main(args): print("Opening TBN file ({})".format(args.tbn_filename)) with LWASVDataFile(args.tbn_filename, ignore_timetag_errors=True) as tbnf: antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) tx_coords = known_transmitters.parse_args(args) if args.visibility_model == 'point': residual_function = point_residual_abs residual_function_chain = None elif args.visibility_model == 'gaussian': residual_function = bind_gaussian_residual(1) residual_function_chain = None elif args.visibility_model == 'chained': residual_function = bind_gaussian_residual(0.5) residual_function_chain = point_residual_abs else: raise RuntimeError("Unknown visibility model option: {args.visibility_model}") if not args.hdf5_file: raise RuntimeError('Please provide an output filename') else: with build_output_file(args.hdf5_file, tbnf, valid_ants, n_baselines, args.integration_length, tx_freq=args.tx_freq, fft_len=args.fft_len, use_pfb=args.use_pfb, use_pol=args.use_pol, opt_method=opt_method, vis_model=args.visibility_model, transmitter_coords=tx_coords) as h5f: # arrays for estimated parameters from each integration l_est = np.array([args.l_guess]) m_est = np.array([args.m_guess]) k = 0 for bl, freqs, vis in compute_visibilities_gen(tbnf, valid_ants, integration_length=args.integration_length, fft_length=args.fft_len, use_pol=args.use_pol, use_pfb=args.use_pfb): # start the optimization at the mean point of the 10 most recent fits if args.visibility_model == 'point': l_init = l_est[-param_guess_av_length:].mean() m_init = m_est[-param_guess_av_length:].mean() else: l_init = 0 m_init = 0 target_bin = np.argmin([abs(args.tx_freq - f) for f in freqs]) # TODO: is this correct? should it be the bin center? uvw = uvw_from_antenna_pairs(bl, wavelength=3e8/args.tx_freq) vis_tbin = vis[:, target_bin] # do the model fitting to get parameter estimates l_out, m_out, opt_result = fit_model_to_vis(uvw, vis_tbin, residual_function, l_init, m_init, export_npy=args.export_npy) nfev = opt_result['nfev'] if residual_function_chain: l_out, m_out, opt_result_chain = fit_model_to_vis(uvw, vis_tbin, residual_function_chain, l_out, m_out, export_npy=args.export_npy) nfev += opt_result_chain['nfev'] cost = opt_result['cost'] # see if we should skip including this in future starting parameter estimates skip = False if args.exclude and k in args.exclude: print("Not including in parameter estimates by request") skip = True if not skip: l_est = np.append(l_est, l_out) m_est = np.append(m_est, m_out) #costs = np.append(costs, cost) # compute source sky location from parameter values src_elev, src_az = lm_to_ea(l_out, m_out) # write data to h5 file h5f['l_start'][k] = l_init h5f['m_start'][k] = m_init h5f['l_est'][k] = l_out h5f['m_est'][k] = m_out h5f['elevation'][k] = src_elev h5f['azimuth'][k] = src_az h5f['cost'][k] = cost h5f['skipped'][k] = skip h5f['nfev'][k] = nfev save_scatter = (args.scatter and k in args.scatter) or (args.scatter_every and k % args.scatter_every == 0) if save_scatter: print("Plotting model and data scatter") vis_phase_scatter_3d(uvw[:,0], uvw[:,1], vis_tbin, show=False, html_savename=f"scatter_{k}.html", l=l_out, m=m_out) k += 1 print("\n\n")