def make_sample_tbn(filename, num_frames=2000000, offset=0): """Takes the defined number of frames and writes them to a new .tbn file Parameters ---------- filename : string name of file to be read (may end in dat, tbn, or nothing) num_frames : int number of frames to be kept (default: 2000000) offset : float number of seconds to skip into file before reading (approximate) """ in_name = os.path.realpath(filename).split('/')[-1] in_base_name = in_name.split('.')[0] out_name = in_base_name + '.tbn' print(f"Reading data from {filename}") print(f"Writing data to {out_name}") if os.path.exists(out_name): raise RuntimeError( f"Output file {out_name} already exists - not going to overwrite it" ) in_tbn = LWASVDataFile(filename) out_fh = open(out_name, 'wb') if offset > 0: t = in_tbn.offset(offset) print(f"Requested offset: {offset} seconds") print(f"Achieved offset: {t} seconds") out_fh.write(in_tbn.fh.read(tbn.FRAME_SIZE * num_frames)) out_fh.close() in_tbn.close() print("\n{} TBN Size: {} kB".format(out_name, os.path.getsize(out_name) / 1024.)) # Check that the datatype is correct according to lsl out_tbn = LWASVDataFile(out_name) print("{} is of type: {} \n".format(out_name, type(out_tbn))) out_tbn.close()
def meta_to_txt(filename, outdir='./', station='lwasv'): """Pulls metadata from TBN file and puts it into a txt file of the same name Parameters ---------- filename : string name of file to be read (may end in dat, tbn, or nothing) outdir : string name of output directory to save textfile in station : string one of 'lwasv' or 'lwa1' for now """ simple_name = filename.split('/')[-1] simple_name = simple_name.split('.')[0] if not outdir.endswith('/'): outdir = outdir + '/' if not pathlib.Path(outdir).exists(): os.mkdir(outdir) print("{} TBN Size: {} kB".format(filename, os.path.getsize(filename) / 1024)) if station == 'lwasv': idfN = LWASVDataFile(filename) simple_name = simple_name + '-LWASV' elif station == 'lwa1': idfN = LWA1DataFile(filename) simple_name = simple_name + '-LWA1' else: raise NotImplementedError( "I haven't implemented that type of station yet") print("{} is of type: {}".format(filename, type(idfN))) # Poll the TBN file for its specifics with open(outdir + simple_name + ".txt", 'w') as meta: meta.write('TBN Metadata:\n') for key, value in idfN.get_info().items(): meta.write(" %s: %s\n" % (str(key), str(value))) idfN.close()
def main(args): # Parse command line options filename = args.filename # Setup the LWA station information if args.metadata is not None: try: station = stations.parse_ssmif(args.metadata) except ValueError: station = metabundleADP.get_station(args.metadata, apply_sdm=True) else: station = stations.lwasv antennas = station.antennas idf = LWASVDataFile(filename) if not isinstance(idf, TBFFile): raise RuntimeError("File '%s' does not appear to be a valid TBF file" % os.path.basename(filename)) jd = idf.get_info('start_time').jd date = idf.get_info('start_time').datetime nFpO = idf.get_info('nchan') // 12 sample_rate = idf.get_info('sample_rate') nInts = idf.get_info('nframe') // nFpO # Get valid stands for both polarizations goodX = [] goodY = [] for i in range(len(antennas)): ant = antennas[i] if ant.combined_status != 33 and not args.all: pass else: if ant.pol == 0: goodX.append(ant) else: goodY.append(ant) # Now combine both lists to come up with stands that # are in both so we can form the cross-polarization # products if we need to good = [] for antX in goodX: for antY in goodY: if antX.stand.id == antY.stand.id: good.append(antX.digitizer - 1) good.append(antY.digitizer - 1) # Report on the valid stands found. This is a little verbose, # but nice to see. print("Found %i good stands to use" % (len(good) // 2, )) for i in good: print("%3i, %i" % (antennas[i].stand.id, antennas[i].pol)) # Number of frames to read in at once and average nFrames = min([int(args.avg_time * sample_rate), nInts]) args.offset = idf.offset(args.offset) nSets = idf.get_info('nframe') // nFpO // nFrames nSets = nSets - int(args.offset * sample_rate) // nFrames central_freq = idf.get_info('freq1') central_freq = central_freq[len(central_freq) // 2] print("Data type: %s" % type(idf)) print("Samples per observations: %i" % nFpO) print("Sampling rate: %i Hz" % sample_rate) print("Tuning frequency: %.3f Hz" % central_freq) print("Captures in file: %i (%.3f s)" % (nInts, nInts / sample_rate)) print("==") print("Station: %s" % station.name) print("Date observed: %s" % date) print("Julian day: %.5f" % jd) print("Offset: %.3f s (%i frames)" % (args.offset, args.offset * sample_rate)) print("Integration Time: %.3f s" % (nFrames / sample_rate)) print("Number of integrations in file: %i" % nSets) # Make sure we don't try to do too many sets if args.samples > nSets: args.samples = nSets # Loop over junks of 100 integrations to make sure that we don't overflow # the FITS IDI memory buffer s = 0 leftToDo = args.samples basename = os.path.split(filename)[1] basename, ext = os.path.splitext(basename) while leftToDo > 0: fitsFilename = "%s.FITS_%i" % ( basename, (s + 1), ) if leftToDo > 100: chunk = 100 else: chunk = leftToDo process_chunk(idf, station, good, fitsFilename, int_time=args.avg_time, pols=args.products, chunk_size=chunk) s += 1 leftToDo = leftToDo - chunk idf.close()
for y, m in enumerate(m_range): cost[x,y] = ls_cost([l, m], u, v, vis) plt.contourf(l_range, m_range, cost) plt.colorbar() plt.xlabel("l") plt.ylabel("m") plt.show() if __name__ == "__main__": plt.close('all') ants, n_baselines = select_antennas(station.antennas, use_pol=0) dfile = LWASVDataFile(tbn_filename) #baselines, visibilities = compute_visibilities(dfile, ants, target_freq) dfile.close() azimuth = station.get_pointing_and_distance(transmitter_coords + [0])[0] bl1d = project_baselines(baselines, azimuth) phases = np.angle(visibilities[0]) vis = visibilities[0] bl2d = np.array([np.array([b[0].stand.x - b[1].stand.x, b[0].stand.y - b[1].stand.y]) for b in baselines]) u = bl2d[:, 0] v = bl2d[:, 1] vis = vis/np.abs(vis)
def main(args): # this first part of the code is run by all processes # set up MPI environment comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if size < 2: raise RuntimeError( f"This program requires at least two MPI processes to function. Please rerun with more resources" ) # designate the last process as the supervisor/file reader supervisor = size - 1 # open the TBN file for reading tbnf = LWASVDataFile(args.tbn_filename, ignore_timetag_errors=True) # figure out the details of the run we want to do tx_coords = known_transmitters.parse_args(args) antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) n_ants = len(valid_ants) sample_rate = tbnf.get_info('sample_rate') # some of our TBNs claim to have frame size 1024 but they are lying frame_size = 512 tbn_center_freq = tbnf.get_info('freq1') total_integrations, _ = compute_integration_numbers( tbnf, args.integration_length) # open the output HDF5 file and create datasets # because of the way parallelism in h5py works all processes (even ones # that don't write to the file) must do this h5f = build_output_file(args.hdf5_file, tbnf, valid_ants, n_baselines, args.integration_length, tx_freq=args.tx_freq, fft_len=args.fft_len, use_pfb=args.use_pfb, use_pol=args.use_pol, opt_method=opt_method, vis_model='gaussian', transmitter_coords=tx_coords, mpi_comm=comm) if rank == supervisor: # the supervisor process runs this code print("supervisor: started") # state info reached_end = False workers_alive = [True for _ in range(size - 1)] int_no = 0 while True: if not reached_end: # grab data for the next available worker try: duration, start_time, data = tbnf.read( args.integration_length) # only use data from valid antennas data = data[[a.digitizer - 1 for a in valid_ants], :] except EOFError: reached_end = True print(f"supervisor: reached EOF") if int_no >= total_integrations: print(f"supervisor: this is the last integration") reached_end = True # get the next "ready" message from the workers st = MPI.Status() msg = comm.recv(status=st) if msg == "ready": print( f"supervisor: received 'ready' message from worker {st.source}" ) # if we're done, send an exit message and mark that we've killed this worker # an empty array indicates that the worker should exit if reached_end: print( f"supervisor: sending exit message to worker {st.source}" ) comm.Send(np.array([]), dest=st.source, tag=int_no) workers_alive[st.source] = False if not any(workers_alive): print(f"supervisor: all workers told to exit, goodbye") break # otherwise, send the data to the worker for processing else: print( f"supervisor: sending data for integration {int_no}/{total_integrations} to worker {st.source}" ) # Send with a capital S is optimized to send numpy arrays comm.Send(data, dest=st.source, tag=int_no) int_no += 1 else: raise ValueError( f"Supervisor received unrecognized message '{msg}' from worker {st.source}" ) tbnf.close() else: # the worker processes run this code print(f"worker {rank} started") # workers don't need access to the TBN file tbnf.close() # figure out the size of the incoming data buffer samples_per_integration = int( round(args.integration_length * sample_rate / frame_size)) * frame_size buffer_shape = (n_ants, samples_per_integration) while True: # send with a lowercase s can send any pickle-able python object # this is a synchronous send - it will block until the message is read by the supervisor # the other sends (e.g. comm.Send) only block until the message is safely taken by MPI, which might happen before the receiver actually reads it comm.ssend("ready", dest=supervisor) # build a buffer to be filled with data data = np.empty(buffer_shape, np.complex64) # receive the data from the supervisor st = MPI.Status() comm.Recv(data, source=supervisor, status=st) int_no = st.tag # if the buffer is empty, we're done if st.count == 0: print(f"worker {rank}: received exit message, exiting") break # otherwise process the data we've recieved print( f"worker {rank}: received data for integration {int_no}, starting processing" ) # run the correlator bl, freqs, vis = fxc.FXMaster( data, valid_ants, LFFT=args.fft_len, pfb=args.use_pfb, sample_rate=sample_rate, central_freq=tbn_center_freq, Pol='xx' if args.use_pol == 0 else 'yy', return_baselines=True, gain_correct=True) # extract the frequency bin we want target_bin = np.argmin([abs(args.tx_freq - f) for f in freqs]) vis_tbin = vis[:, target_bin] # baselines in wavelengths uvw = uvw_from_antenna_pairs(bl, wavelength=3e8 / args.tx_freq) # model fitting l_out, m_out, opt_result = fit_model_to_vis(uvw, vis_tbin, residual_function, l_init, m_init, verbose=False) # convert direction cosines to sky coords src_elev, src_az = lm_to_ea(l_out, m_out) # write data to h5 file h5f['l_start'][int_no] = l_init h5f['m_start'][int_no] = m_init h5f['l_est'][int_no] = l_out h5f['m_est'][int_no] = m_out h5f['elevation'][int_no] = src_elev h5f['azimuth'][int_no] = src_az h5f['cost'][int_no] = opt_result['cost'] h5f['nfev'][int_no] = opt_result['nfev'] # compute the bin power and save it to the file # arbitrarily picking the tenth antenna in this list power_calc_data = data[10, :] h5f['snr_est'][int_no] = estimate_snr(power_calc_data, args.fft_len, args.tx_freq, sample_rate, tbn_center_freq) print(f"worker {rank}: done processing integration {int_no}") # back to common code for both supervisor and workers h5f.attrs['total_integrations'] = int_no h5f.close()
def main(args): # this first part of the code is run by all processes # set up MPI environment comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if size < 2: raise RuntimeError( f"This program requires at least two MPI processes to function. Please rerun with more resources" ) # designate the last process as the supervisor/file reader supervisor = size - 1 # open the TBN file for reading tbnf = LWASVDataFile(args.tbn_filename, ignore_timetag_errors=True) # figure out the details of the run we want to do tx_coords = known_transmitters.parse_args(args) antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) n_ants = len(valid_ants) total_integrations, _ = compute_integration_numbers( tbnf, args.integration_length) sample_rate = tbnf.get_info('sample_rate') # some of our TBNs claim to have frame size 1024 but they are lying frame_size = 512 tbn_center_freq = tbnf.get_info('freq1') # open the output HDF5 file and create datasets # because of the way parallelism in h5py works all processes (even ones # that don't write to the file) must do this h5f = build_output_file(args.hdf5_file, tbnf, valid_ants, n_baselines, args.integration_length, tx_freq=args.tx_freq, fft_len=args.fft_len, use_pfb=args.use_pfb, use_pol=args.use_pol, transmitter_coords=tx_coords, mpi_comm=comm) if args.point_finding_alg == 'all' or args.point_finding_alg == 'peak': h5f.create_dataset_like('l_peak', h5f['l_est']) h5f.create_dataset_like('m_peak', h5f['m_est']) h5f.create_dataset_like('elevation_peak', h5f['elevation']) h5f.create_dataset_like('azimuth_peak', h5f['azimuth']) if args.point_finding_alg == 'all' or args.point_finding_alg == 'CoM': h5f.create_dataset_like('l_CoM', h5f['l_est']) h5f.create_dataset_like('m_CoM', h5f['m_est']) h5f.create_dataset_like('elevation_CoM', h5f['elevation']) h5f.create_dataset_like('azimuth_CoM', h5f['azimuth']) else: raise NotImplementedError( f"Unrecognized point finding algorithm: {args.point_finding_alg}") del h5f['l_est'] del h5f['m_est'] del h5f['elevation'] del h5f['azimuth'] if rank == supervisor: # the supervisor process runs this code print("supervisor: started") # state info reached_end = False workers_alive = [True for _ in range(size - 1)] int_no = 0 while True: if not reached_end: # grab data for the next available worker try: duration, start_time, data = tbnf.read( args.integration_length) # only use data from valid antennas data = data[[a.digitizer - 1 for a in valid_ants], :] except EOFError: reached_end = True print(f"supervisor: reached EOF") if int_no >= total_integrations: print(f"supervisor: this is the last integration") reached_end = True # get the next "ready" message from the workers st = MPI.Status() msg = comm.recv(status=st) if msg == "ready": print( f"supervisor: received 'ready' message from worker {st.source}" ) # if we're done, send an exit message and mark that we've killed this worker # an empty array indicates that the worker should exit if reached_end: print( f"supervisor: sending exit message to worker {st.source}" ) comm.Send(np.array([]), dest=st.source, tag=int_no) workers_alive[st.source] = False if not any(workers_alive): print(f"supervisor: all workers told to exit, goodbye") break # otherwise, send the data to the worker for processing else: print( f"supervisor: sending data for integration {int_no}/{total_integrations} to worker {st.source}" ) # Send with a capital S is optimized to send numpy arrays comm.Send(data, dest=st.source, tag=int_no) int_no += 1 else: raise ValueError( f"Supervisor received unrecognized message '{msg}' from worker {st.source}" ) tbnf.close() else: # the worker processes run this code print(f"worker {rank} started") # workers don't need access to the TBN file tbnf.close() # figure out the size of the incoming data buffer samples_per_integration = int( round(args.integration_length * sample_rate / frame_size)) * frame_size buffer_shape = (n_ants, samples_per_integration) while True: # send with a lowercase s can send any pickle-able python object # this is a synchronous send - it will block until the message is read by the supervisor # the other sends (e.g. comm.Send) only block until the message is safely taken by MPI, which might happen before the receiver actually reads it comm.ssend("ready", dest=supervisor) # build a buffer to be filled with data data = np.empty(buffer_shape, np.complex64) # receive the data from the supervisor st = MPI.Status() comm.Recv(data, source=supervisor, status=st) int_no = st.tag # if the buffer is empty, we're done if st.count == 0: print(f"worker {rank}: received exit message, exiting") break # otherwise process the data we've recieved print( f"worker {rank}: received data for integration {int_no}, starting processing" ) # run the correlator bl, freqs, vis = fxc.FXMaster( data, valid_ants, LFFT=args.fft_len, pfb=args.use_pfb, sample_rate=sample_rate, central_freq=tbn_center_freq, Pol='xx' if args.use_pol == 0 else 'yy', return_baselines=True, gain_correct=True) gridded_image = grid_visibilities(bl, freqs, vis, args.tx_freq, station) save_all_sky = (args.all_sky and int_no in args.all_sky) or ( args.all_sky_every and int_no % args.all_sky_every == 0) if args.point_finding_alg == 'all' or 'peak': result = get_gimg_max(gridded_image, return_img=save_all_sky) l = result[0] m = result[1] src_elev, src_az = lm_to_ea(l, m) h5f['l_peak'][int_no] = l h5f['m_peak'][int_no] = m h5f['elevation_peak'][int_no] = src_elev h5f['azimuth_peak'][int_no] = src_az if args.point_finding_alg == 'all' or args.point_finding_alg == 'CoM': result = get_gimg_center_of_mass(gridded_image, return_img=save_all_sky) l = result[0] m = result[1] src_elev, src_az = lm_to_ea(l, m) h5f['l_CoM'][int_no] = l h5f['m_CoM'][int_no] = m h5f['elevation_CoM'][int_no] = src_elev h5f['azimuth_CoM'][int_no] = src_az if save_all_sky: img = result[2] extent = result[3] fig, ax = plt.subplots() ax.imshow(img, extent=extent, origin='lower', interpolation='nearest') plt.savefig('allsky_int_{}.png'.format(int_no)) # compute the bin power and save it to the file # arbitrarily picking the tenth antenna in this list power_calc_data = data[10, :] h5f['snr_est'][int_no] = estimate_snr(power_calc_data, args.fft_len, args.tx_freq, sample_rate, tbn_center_freq) print(f"worker {rank}: done processing integration {int_no}") # back to common code for both supervisor and workers h5f.attrs['total_integrations'] = int_no h5f.close()