def get_optimal_delay_padding(antennaSet1, antennaSet2, LFFT=64, sample_rate=None, central_freq=0.0, pol='XX', phase_center='z'): # Decode the polarization product into something that we can use to figure # out which antennas to use for the cross-correlation if pol == '*': antennas1 = antennaSet1 antennas2 = antennaSet2 else: pol1, pol2 = pol_to_pols(pol) antennas1 = [a for a in antennaSet1 if a.pol == pol1] antennas2 = [a for a in antennaSet2 if a.pol == pol1] # Combine the two sets and proceede antennas1.extend(antennas2) nStands = len(antennas1) # Create a reasonable mock setup for computing the delays if sample_rate is None: sample_rate = dp_common.fS freq = numpy.fft.fftfreq(LFFT, d=1.0/sample_rate) freq += float(central_freq) freq = numpy.fft.fftshift(freq) # Get the location of the phase center in radians and create a # pointing vector if phase_center == 'z': azPC = 0.0 elPC = numpy.pi/2.0 else: if isinstance(phase_center, ephem.Body): azPC = phase_center.az * 1.0 elPC = phase_center.alt * 1.0 elif isinstance(phase_center, AstroAltAz): azPC = phase_center.az.radian elPC = phase_center.alt.radian else: azPC = phase_center[0]*numpy.pi/180.0 elPC = phase_center[1]*numpy.pi/180.0 source = numpy.array([numpy.cos(elPC)*numpy.sin(azPC), numpy.cos(elPC)*numpy.cos(azPC), numpy.sin(elPC)]) # Define the cable/signal delay caches to help correlate along and compute # the delays that we need to apply to align the signals dlyRef = len(freq)//2 delays1 = numpy.zeros((nStands,LFFT)) for i in list(range(nStands)): xyz1 = numpy.array([antennas1[i].stand.x, antennas1[i].stand.y, antennas1[i].stand.z]) delays1[i,:] = antennas1[i].cable.delay(freq) - numpy.dot(source, xyz1) / vLight minDelay = delays1[:,dlyRef].min() # Round to the next lowest 5 us, negate, and return minDelay = numpy.floor( minDelay / 5e-6) * 5e-6 return -minDelay
def main(args): h5fi = h5py.File(args.input_file, 'r') h5fo = h5py.File(args.output_file,'w') # Copy over important attributes for key in h5fi.attrs.keys(): h5fo.attrs[key]=h5fi.attrs[key] # Copy over important datasets for key in h5fi.keys(): temp_arr = h5fi[key] h5fo.create_dataset('vis_{}'.format(key),data=temp_arr) h5fo.attrs['grid_size']=args.size h5fo.attrs['grid_res']=args.res h5fo.attrs['grid_wres']=args.wres h5fo.create_dataset('l_est', (len(h5fo['vis_l_est']),)) h5fo.create_dataset('m_est', (len(h5fo['vis_l_est']),)) h5fo.create_dataset('extent', (len(h5fo['vis_l_est']),4)) h5fo.create_dataset('elevation', (len(h5fo['vis_l_est']),)) h5fo.create_dataset('azimuth', (len(h5fo['vis_l_est']),)) h5fo.create_dataset('height', (len(h5fo['vis_l_est']),)) h5fi.close() # done with input data now ## Begin doing stuff antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, h5fo.attrs['use_pol'], exclude=[256]) # to exclude outrigger tx_coords = h5fo.attrs['tx_coordinates'] rx_coords = [station.lat * 180/np.pi, station.lon * 180/np.pi] ## Build freqs (same for every 'integration') freqs = np.empty((h5fo.attrs['fft_len'],),dtype=np.float64) #! Need to think of intelligent way of doing this. #! target_bin will probably not matter since all vis is the same freqs5 = [5284999.9897182, 5291249.9897182, 5297499.9897182, 5303749.9897182, 5309999.9897182, 5316249.9897182, 5322499.9897182, 5328749.9897182, 5334999.9897182, 5341249.9897182, 5347499.9897182, 5353749.9897182, 5359999.9897182, 5366249.9897182, 5372499.9897182, 5378749.9897182] for i in range(len(freqs)): freqs[i]=freqs5[i] ## Build bl (same for every 'integration') pol_string = 'xx' if h5fo.attrs['use_pol'] == 0 else 'yy' pol1, pol2 = pol_to_pols(pol_string) antennas1 = [a for a in valid_ants if a.pol == pol1] antennas2 = [a for a in valid_ants if a.pol == pol2] nStands = len(antennas1) baselines = uvutils.get_baselines(antennas1, antennas2=antennas2, include_auto=False, indicies=True) antennaBaselines = [] for bl in range(len(baselines)): antennaBaselines.append( (antennas1[baselines[bl][0]], antennas2[baselines[bl][1]]) ) bl = antennaBaselines uvw_m = np.array([np.array([b[0].stand.x - b[1].stand.x, b[0].stand.y - b[1].stand.y, b[0].stand.z - b[1].stand.z]) for b in bl]) uvw = np.empty((len(bl), 3, len(freqs))) for i, f in enumerate(freqs): # wavelength = 3e8/f # TODO this should be fixed. What is currently happening is not true. Well it is, but only if you're looking for a specific transmitter frequency. Which I guess we are. I just mean it's not generalized. wavelength = 3e8/h5fo.attrs['tx_freq'] uvw[:,:,i] = uvw_m/wavelength # Build antenna array (gets used in the VisibilityDataSet) # jd can't matter, right? jd = 2458847.2362531545 antenna_array = simVis.build_sim_array(station, valid_ants, freqs/1e9, jd=jd, force_flat=True) # we only want the bin nearest to our frequency target_bin = np.argmin([abs(h5fo.attrs['tx_freq'] - f) for f in freqs]) # Needed for PolarizationDataSet if h5fo.attrs['use_pol'] == 0: pol_string = 'XX' p=0 # this is related to the enumerate in lsl.imaging.utils.CorrelatedIDI().get_data_set() (for when there are multiple pols in a single dataset) else: raise RuntimeError("Only pol. XX supported right now.") if args.all_sky: fig, ax = plt.subplots() for k in np.arange(len(h5fo['vis_l_est'])): l_in = h5fo['vis_l_est'][k] m_in = h5fo['vis_m_est'][k] ## Build vis vismodel = point_source_visibility_model_uv(uvw[:,0,0],uvw[:,1,0],l_in,m_in) vis = np.empty((len(vismodel), len(freqs)), dtype=np.complex64) for i in np.arange(vis.shape[1]): vis[:,i] = vismodel if args.export_npy: print(args.export_npy) print("Exporting modelled u, v, w, and visibility") np.save('model-uvw{}.npy'.format(k), uvw) np.save('model-vis{}.npy'.format(k), vis) ## Start to build up the data structure for VisibilityDataSet dataSet = VisibilityDataSet(jd=jd, freq=freqs, baselines=bl, uvw=uvw, antennarray=antenna_array) polDataSet = PolarizationDataSet(pol_string, data=vis) dataSet.append(polDataSet) print('| Gridding and imaging with size={}, res={}, wres={}'.format(args.size, args.res, args.wres)) gridded_image = build_gridded_image(dataSet, pol=pol_string, chan=target_bin, size=args.size, res=args.res, wres=args.wres) if args.export_npy: print("Exporting gridded u, v, and visibility") u,v = gridded_image.get_uv() np.save('gridded-u{}.npy'.format(k), u) np.save('gridded-v{}.npy'.format(k), v) np.save('gridded-vis{}.npy'.format(k), gridded_image.uv) l,m,img,extent=get_gimg_max(gridded_image, return_img=True) # Compute other values of interest elev, az = lm_to_ea(l, m) height = flatmirror_height(tx_coords, rx_coords, elev) h5fo['l_est'][k] = l h5fo['m_est'][k] = m h5fo['extent'][k] = extent h5fo['elevation'][k] = elev h5fo['azimuth'][k] = az h5fo['height'][k] = height if args.all_sky: ax.imshow(img, extent=extent, origin='lower', interpolation='nearest') ax.set_title('size={}, res={}, wres={}, iteration={}'.format(args.size,args.res,args.wres,k)) ax.set_xlabel('l') ax.set_ylabel('m') ax.plot(l,m,marker='o', color='k', label='Image Max.') ax.plot(l_in,m_in,marker='x', color='r', label='Model (input)') plt.legend(loc='lower right') plt.savefig('allsky{}.png'.format(k)) plt.cla() save_pkl_gridded = args.pkl_gridded and k in args.pkl_gridded if save_pkl_gridded: quickDict={'image':img, 'extent':extent} with open('gridded{}.pkl'.format(k), 'wb') as f: pickle.dump(quickDict, f, protocol=pickle.HIGHEST_PROTOCOL) h5fo.close()
def process_chunk(idf, site, good, filename, int_time=5.0, pols=[ 'xx', ], chunk_size=100): """ Given a lsl.reader.ldp.TBNFile instances and various parameters for the cross-correlation, write cross-correlate the data and save it to a file. """ # Get antennas antennas = site.antennas # Get the metadata sample_rate = idf.get_info('sample_rate') freq = idf.get_info('freq1') # Create the list of good digitizers and a digitizer to Antenna instance mapping. # These are: # toKeep -> mapping of digitizer number to array location # mapper -> mapping of Antenna instance to array location toKeep = [antennas[i].digitizer - 1 for i in good] mapper = [antennas[i] for i in good] # Create a list of unqiue stands to know what style of IDI file to create stands = set([antennas[i].stand.id for i in good]) # Main loop over the input file to read in the data and organize it. Several control # variables are defined for this: # ref_time -> time (in seconds since the UNIX epoch) for the first data set # setTime -> time (in seconds since the UNIX epoch) for the current data set ref_time = 0.0 setTime = 0.0 wallTime = time.time() for s in range(chunk_size): try: readT, t, data = idf.read(int_time) except Exception as e: print("Error: %s" % str(e)) continue ## Prune out what we don't want data = data[toKeep, :, :] ## Split the polarizations antennasX, antennasY = [ a for i, a in enumerate(antennas) if a.pol == 0 and i in toKeep ], [a for i, a in enumerate(antennas) if a.pol == 1 and i in toKeep] dataX, dataY = data[0::2, :, :], data[1::2, :, :] validX = numpy.ones((dataX.shape[0], dataX.shape[2]), dtype=numpy.uint8) validY = numpy.ones((dataY.shape[0], dataY.shape[2]), dtype=numpy.uint8) ## Apply the cable delays as phase rotations for i in range(dataX.shape[0]): gain = numpy.sqrt(antennasX[i].cable.gain(freq)) phaseRot = numpy.exp(2j*numpy.pi*freq*(antennasX[i].cable.delay(freq) \ -antennasX[i].stand.z/speedOfLight)) for j in range(dataX.shape[2]): dataX[i, :, j] *= phaseRot / gain for i in range(dataY.shape[0]): gain = numpy.sqrt(antennasY[i].cable.gain(freq)) phaseRot = numpy.exp(2j*numpy.pi*freq*(antennasY[i].cable.delay(freq)\ -antennasY[i].stand.z/speedOfLight)) for j in range(dataY.shape[2]): dataY[i, :, j] *= phaseRot / gain setTime = t if s == 0: ref_time = setTime # Setup the set time as a python datetime instance so that it can be easily printed setDT = setTime.datetime print("Working on set #%i (%.3f seconds after set #1 = %s)" % ((s + 1), (setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f"))) # Loop over polarization products for pol in pols: print("-> %s" % pol) if pol[0] == 'x': a1, d1, v1 = antennasX, dataX, validX else: a1, d1, v1 = antennasY, dataY, validY if pol[1] == 'x': a2, d2, v2 = antennasX, dataX, validX else: a2, d2, v2 = antennasY, dataY, validY ## Get the baselines baselines = uvutils.get_baselines(a1, antennas2=a2, include_auto=True, indicies=True) blList = [] for bl in range(len(baselines)): blList.append((a1[baselines[bl][0]], a2[baselines[bl][1]])) ## Run the cross multiply and accumulate vis = XEngine2(d1, d2, v1, v2) # Select the right range of channels to save toUse = numpy.where((freq > 5.0e6) & (freq < 93.0e6)) toUse = toUse[0] # If we are in the first polarazation product of the first iteration, setup # the FITS IDI file. if s == 0 and pol == pols[0]: pol1, pol2 = fxc.pol_to_pols(pol) if len(stands) > 255: fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time) else: fits = fitsidi.Idi(filename, ref_time=ref_time) fits.set_stokes(pols) fits.set_frequency(freq[toUse]) fits.set_geometry(site, [a for a in mapper if a.pol == pol1]) # Convert the setTime to a MJD and save the visibilities to the FITS IDI file obsTime = astro.unix_to_taimjd(setTime) fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol) print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" % ((time.time() - wallTime), (time.time() - wallTime) / (s + 1))) # Cleanup after everything is done fits.write() fits.close() del (fits) del (data) del (vis) return True
def process_chunk(idf, site, good, filename, int_time=5.0, LFFT=64, overlap=1, pfb=False, pols=[ 'xx', ], chunk_size=100): """ Given a lsl.reader.ldp.TBNFile instances and various parameters for the cross-correlation, write cross-correlate the data and save it to a file. """ # Get antennas antennas = [] for a in site.antennas: if a.digitizer != 0: antennas.append(a) # Get the metadata sample_rate = idf.get_info('sample_rate') central_freq = idf.get_info('freq1') # Create the list of good digitizers and a digitizer to Antenna instance mapping. # These are: # toKeep -> mapping of digitizer number to array location # mapper -> mapping of Antenna instance to array location toKeep = [antennas[i].digitizer - 1 for i in good] mapper = [antennas[i] for i in good] # Create a list of unqiue stands to know what style of IDI file to create stands = set([antennas[i].stand.id for i in good]) # Main loop over the input file to read in the data and organize it. Several control # variables are defined for this: # ref_time -> time (in seconds since the UNIX epoch) for the first data set # setTime -> time (in seconds since the UNIX epoch) for the current data set ref_time = 0.0 setTime = 0.0 wallTime = time.time() for s in range(chunk_size): try: readT, t, data = idf.read(int_time) except Exception as e: print("Error: %s" % str(e)) continue ## Prune out what we don't want data = data[toKeep, :] setTime = t if s == 0: ref_time = setTime # Setup the set time as a python datetime instance so that it can be easily printed setDT = datetime.utcfromtimestamp(setTime) setDT.replace(tzinfo=UTC()) print("Working on set #%i (%.3f seconds after set #1 = %s)" % ((s + 1), (setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f"))) # Loop over polarization products for pol in pols: print("-> %s" % pol) blList, freq, vis = fxc.FXMaster(data, mapper, LFFT=LFFT, overlap=overlap, pfb=pfb, include_auto=True, verbose=False, sample_rate=sample_rate, central_freq=central_freq, pol=pol, return_baselines=True, gain_correct=True) # Select the right range of channels to save toUse = numpy.where((freq > 5.0e6) & (freq < 93.0e6)) toUse = toUse[0] # If we are in the first polarazation product of the first iteration, setup # the FITS IDI file. if s == 0 and pol == pols[0]: pol1, pol2 = fxc.pol_to_pols(pol) if len(stands) > 255: fits = fitsidi.ExtendedIdi(filename, ref_time=ref_time) else: fits = fitsidi.Idi(filename, ref_time=ref_time) fits.set_stokes(pols) fits.set_frequency(freq[toUse]) fits.set_geometry(site, [a for a in mapper if a.pol == pol1]) # Convert the setTime to a MJD and save the visibilities to the FITS IDI file obsTime = astro.unix_to_taimjd(setTime) fits.add_data_set(obsTime, readT, blList, vis[:, toUse], pol=pol) print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" % ((time.time() - wallTime), (time.time() - wallTime) / (s + 1))) # Cleanup after everything is done fits.write() fits.close() del (fits) del (data) del (vis) return True
def fengine(signals, antennas, LFFT=64, overlap=1, include_auto=False, verbose=False, window=null_window, sample_rate=None, central_freq=0.0, pol='XX', gain_correct=False, return_baselines=False, clip_level=0, phase_center='z', delayPadding=40e-6): """ Multi-rate F engine based on the lsl.correlator.fx.FXMaster() function. """ # Decode the polarization product into something that we can use to figure # out which antennas to use for the cross-correlation if pol == '*': antennas1 = antennas signalsIndex1 = [i for (i, a) in enumerate(antennas)] else: pol1, pol2 = pol_to_pols(pol) antennas1 = [a for a in antennas if a.pol == pol1] signalsIndex1 = [i for (i, a) in enumerate(antennas) if a.pol == pol1] nStands = len(antennas1) # Figure out if we are working with complex (I/Q) data or only real. This # will determine how the FFTs are done since the real data mirrors the pos- # itive and negative Fourier frequencies. if signals.dtype.kind == 'c': lFactor = 1 doFFTShift = True central_freq = float(central_freq) else: lFactor = 2 doFFTShift = False if sample_rate is None: sample_rate = dp_common.fS freq = numpy.fft.fftfreq(lFactor*LFFT, d=1.0/sample_rate) + central_freq if doFFTShift: freq = numpy.fft.fftshift(freq) freq = freq[:LFFT] # Get the location of the phase center in radians and create a # pointing vector if phase_center == 'z': azPC = 0.0 elPC = numpy.pi/2.0 else: if isinstance(phase_center, ephem.Body): azPC = phase_center.az * 1.0 elPC = phase_center.alt * 1.0 elif isinstance(phase_center, AstroAltAz): azPC = phase_center.az.radian elPC = phase_center.alt.radian else: azPC = phase_center[0]*numpy.pi/180.0 elPC = phase_center[1]*numpy.pi/180.0 source = numpy.array([numpy.cos(elPC)*numpy.sin(azPC), numpy.cos(elPC)*numpy.cos(azPC), numpy.sin(elPC)]) # Define the cable/signal delay caches to help correlate along and compute # the delays that we need to apply to align the signals dlyRef = len(freq)//2 delays1 = numpy.zeros((nStands,LFFT)) for i in list(range(nStands)): try: xyz1 = numpy.array([antennas1[i].apparent_stand.x, antennas1[i].apparent_stand.y, antennas1[i].apparent_stand.z]) except AttributeError: xyz1 = numpy.array([antennas1[i].stand.x, antennas1[i].stand.y, antennas1[i].stand.z]) delays1[i,:] = antennas1[i].cable.delay(freq) - numpy.dot(source, xyz1) / vLight + delayPadding minDelay = delays1[:,dlyRef].min() if minDelay < 0: raise RuntimeError('Minimum data stream delay is negative: %.3f us' % (minDelay*1e6,)) # F - defaults to running parallel in C via OpenMP if len(signalsIndex1) != signals.shape[0]: signalsF1, validF1 = _core.FEngine(signals[signalsIndex1,:], freq, delays1, LFFT=LFFT, overlap=overlap, sample_rate=sample_rate, clip_level=clip_level, window=window) else: signalsF1, validF1 = _core.FEngine(signals, freq, delays1, LFFT=LFFT, overlap=overlap, sample_rate=sample_rate, clip_level=clip_level, window=window) return freq, signalsF1, validF1, delays1
def main(args): ## Check we should bother doing anything if not args.export_npy and not args.export_h5 and not args.all_sky and not args.pkl_gridded: raise RuntimeError( "You have not selected a data output of any type. Read the docstring and pick something for me to do." ) # Normalize all inputs to the same length sizes = [int(item) for item in args.size.split(',')] reses = [float(item) for item in args.res.split(',')] wreses = [float(item) for item in args.wres.split(',')] maxinputlen = max(len(sizes), len(reses), len(wreses)) if len(sizes) not in [1, maxinputlen] or len(reses) not in [ 1, maxinputlen ] or len(wreses) not in [1, maxinputlen]: raise RuntimeError(" \ For size, res and wres you must pass either the same number of values as the max or a single value.\n \ For example:\n \ ALLOWED -> sizes=175,180,190, res=0.5, wres=0.5\n \ -> sizes=175,180,190, res=0.5, wres=0.5,0.6,0.7\n \ NOT ALLOWED -> sizes=175,180,190, res=0.5, wres=0.5,0.6 \ ") if len( sizes ) != maxinputlen: # You'd think there must be a good way to do this with list comprehension. sizes = sizes * maxinputlen if len(reses) != maxinputlen: reses = reses * maxinputlen if len(wreses) != maxinputlen: wreses = wreses * maxinputlen all_grid_params = [] while len(sizes) > 0: all_grid_params.append({ 'size': sizes.pop(), 'res': reses.pop(), 'wres': wreses.pop() }) ## Begin doing stuff tx_coords = known_transmitters.parse_args(args) if not transmitter_coords: print("Please specify a transmitter location") return rx_coords = [station.lat * 180 / np.pi, station.lon * 180 / np.pi] antennas = station.antennas valid_ants, n_baselines = select_antennas(antennas, args.use_pol) if args.export_h5: h5fname = "simulation-results.h5" print("Output will be written to {}".format(h5fname)) h5f = h5py.File(h5fname, 'w') ats = h5f.attrs ats['transmitter'] = args.transmitter ats['tx_freq'] = args.tx_freq ats['valid_ants'] = [a.id for a in valid_ants] ats['n_baselines'] = n_baselines ats['fft_len'] = args.fft_len ats['use_pol'] = args.use_pol ats['int_length'] = args.integration_length ats['l_model'] = args.l_model ats['m_model'] = args.m_model h5f.create_dataset('l_est', (len(all_grid_params), )) h5f.create_dataset('m_est', (len(all_grid_params), )) h5f.create_dataset('wres', (len(all_grid_params), )) h5f.create_dataset('res', (len(all_grid_params), )) h5f.create_dataset('size', (len(all_grid_params), )) h5f.create_dataset('extent', (len(all_grid_params), 4)) h5f.create_dataset('elevation', (len(all_grid_params), )) h5f.create_dataset('azimuth', (len(all_grid_params), )) h5f.create_dataset('height', (len(all_grid_params), )) ## Build freqs freqs = np.empty((args.fft_len, ), dtype=np.float64) #! Need to think of intelligent way of doing this. #! target_bin will probably not matter since all vis is the same freqs5 = [ 5284999.9897182, 5291249.9897182, 5297499.9897182, 5303749.9897182, 5309999.9897182, 5316249.9897182, 5322499.9897182, 5328749.9897182, 5334999.9897182, 5341249.9897182, 5347499.9897182, 5353749.9897182, 5359999.9897182, 5366249.9897182, 5372499.9897182, 5378749.9897182 ] for i in range(len(freqs)): freqs[i] = freqs5[i] ## Build bl pol_string = 'xx' if args.use_pol == 0 else 'yy' pol1, pol2 = pol_to_pols(pol_string) antennas1 = [a for a in valid_ants if a.pol == pol1] antennas2 = [a for a in valid_ants if a.pol == pol2] nStands = len(antennas1) baselines = uvutils.get_baselines(antennas1, antennas2=antennas2, include_auto=False, indicies=True) antennaBaselines = [] for bl in range(len(baselines)): antennaBaselines.append( (antennas1[baselines[bl][0]], antennas2[baselines[bl][1]])) bl = antennaBaselines uvw_m = np.array([ np.array([ b[0].stand.x - b[1].stand.x, b[0].stand.y - b[1].stand.y, b[0].stand.z - b[1].stand.z ]) for b in bl ]) uvw = np.empty((len(bl), 3, len(freqs))) for i, f in enumerate(freqs): # wavelength = 3e8/f # TODO this should be fixed. What is currently happening is not true. Well it is, but only if you're looking for a specific transmitter frequency. Which I guess we are. I just mean it's not generalized. wavelength = 3e8 / args.tx_freq uvw[:, :, i] = uvw_m / wavelength ## Build vis vismodel = point_source_visibility_model_uv(uvw[:, 0, 0], uvw[:, 1, 0], args.l_model, args.m_model) vis = np.empty((len(vismodel), len(freqs)), dtype=np.complex64) for i in np.arange(vis.shape[1]): vis[:, i] = vismodel if args.export_npy: print(args.export_npy) print("Exporting modelled u, v, w, and visibility") np.save('model-uvw.npy', uvw) np.save('model-vis.npy', vis) ## Start to build up the data structure for VisibilityDataSet # we only want the bin nearest to our frequency target_bin = np.argmin([abs(args.tx_freq - f) for f in freqs]) # This can't matter, right? # jd = tbnf.get_info('start_time').jd jd = 2458847.2362531545 # Build antenna array antenna_array = simVis.build_sim_array(station, antennas, freqs / 1e9, jd=jd, force_flat=True) dataSet = VisibilityDataSet(jd=jd, freq=freqs, baselines=bl, uvw=uvw, antennarray=antenna_array) if args.use_pol == 0: pol_string = 'XX' p = 0 # this is related to the enumerate in lsl.imaging.utils.CorrelatedIDI().get_data_set() (for when there are multiple pols in a single dataset) else: raise RuntimeError("Only pol. XX supported right now.") polDataSet = PolarizationDataSet(pol_string, data=vis) dataSet.append(polDataSet) if args.all_sky: fig, ax = plt.subplots() # Iterate over size/res/wres and generate multiple grids/images k = 0 for grid_params in all_grid_params: print('| Gridding and imaging with size={}, res={}, wres={}'.format( grid_params['size'], grid_params['res'], grid_params['wres'])) gridded_image = build_gridded_image(dataSet, pol=pol_string, chan=target_bin, size=grid_params['size'], res=grid_params['res'], wres=grid_params['wres']) if args.export_npy: print("Exporting gridded u, v, and visibility") u, v = gridded_image.get_uv() np.save( 'gridded-u-size-{}-res-{}-wres-{}.npy'.format( grid_params['size'], grid_params['res'], grid_params['wres']), u) np.save( 'gridded-v-size-{}-res-{}-wres-{}.npy'.format( grid_params['size'], grid_params['res'], grid_params['wres']), v) np.save( 'gridded-vis-size-{}-res-{}-wres-{}.npy'.format( grid_params['size'], grid_params['res'], grid_params['wres']), gridded_image.uv) l, m, img, extent = get_gimg_max(gridded_image, return_img=True) # Compute other values of interest elev, az = lm_to_ea(l, m) height = flatmirror_height(tx_coords, rx_coords, elev) if args.export_h5: h5f['l_est'][k] = l h5f['m_est'][k] = m h5f['wres'][k] = grid_params['wres'] h5f['res'][k] = grid_params['res'] h5f['size'][k] = grid_params['size'] h5f['extent'][k] = extent h5f['elevation'][k] = elev h5f['azimuth'][k] = az h5f['height'][k] = height if args.all_sky: ax.imshow(img, extent=extent, origin='lower', interpolation='nearest') ax.set_title('size={}, res={}, wres={}'.format( grid_params['size'], grid_params['res'], grid_params['wres'])) ax.set_xlabel('l') ax.set_ylabel('m') ax.plot(l, m, marker='o', color='k', label='Image Max.') ax.plot(args.l_model, args.m_model, marker='x', color='r', label='Model (input)') plt.legend(loc='lower right') plt.savefig('allsky_size_{}_res_{}_wres_{}.png'.format( grid_params['size'], grid_params['res'], grid_params['wres'])) plt.cla() save_pkl_gridded = args.pkl_gridded and k in args.pkl_gridded if save_pkl_gridded: quickDict = {'image': img, 'extent': extent} with open( 'gridded_size_{}_res_{}_wres_{}.pkl'.format( grid_params['size'], grid_params['res'], grid_params['wres']), 'wb') as f: pickle.dump(quickDict, f, protocol=pickle.HIGHEST_PROTOCOL) k += 1 if args.export_h5: h5f.close()
def simulate_visibilities_gen(model, model_params, freqs, antennas=stations.lwasv.antennas, pol='XX', noise_sigma=None): ''' Returns a generator which provides simulated visibilities according to a specified model. Parameters: model: a function that takes as arugments - u : a np.array of u coordinates - v : a np.array of v coordinates - some number of parameters (e.g. l, m) and returns an np.array the same size as the u and v coordinate vectors containing the visibility samples from the model at the (u,v) points. model_params: a list of tuples, each containing values for the scalar parameters of model. Each tuple will be used to call model in a subsequent iteration of the generator. freqs: a list of frequencies. for now these are just used for baseline calculation and not passed into the model. TODO: pass freqs to the model ants: a list of lsl antenna objects the baselines of which will be used to generate the (u,v) coordinate vectors Returns: A generator yielding a tuple of (baselines, freqs, visibilities) - baselines: a list of pairs of antenna objects with each pair representing a baseline - freqs: same as the argument freqs - visibilities: a numpy array of visibility samples corresponding to the antenna pairs in baselines for each frequency in freqs The generator will yield a tuple for each set of parameters in model_params. ''' print("Simulating visibilities") print(f"| using model {model.__name__}") print( f"| received {len(model_params)} sets of parameters, will emit that many sets of visibilities" ) pol1, pol2 = fxc.pol_to_pols(pol) antennas1 = [a for a in antennas if a.pol == pol1] antennas2 = [a for a in antennas if a.pol == pol2] baseline_indices = uvutils.get_baselines(antennas1, antennas2=antennas2, include_auto=False, indicies=True) baselines = [] for bl in range(len(baseline_indices)): baselines.append((antennas1[baseline_indices[bl][0]], antennas2[baseline_indices[bl][1]])) for params in model_params: visibilities = np.empty((len(baselines), len(freqs)), dtype=np.complex128) for k, freq in enumerate(freqs): wl = 3e8 / freq uvw = uvw_from_antenna_pairs(baselines, wl) u = uvw[:, 0] v = uvw[:, 1] w = uvw[:, 2] visibilities[:, k] = model(u, v, w, *params) if noise_sigma is not None: noise = np.random.normal( 0, noise_sigma, len(visibilities)) + 1j * np.random.normal( 0, noise_sigma, len(visibilities)) visibilities[:, k] += noise yield baselines, freqs, visibilities return
def process_chunk(idf, site, good, filename, LFFT=64, overlap=1, pfb=False, pols=['xx', 'yy']): """ Given an lsl.reader.ldp.TBWFile instances and various parameters for the cross-correlation, write cross-correlate the data and save it to a file. """ # Get antennas antennas = site.antennas # Get the metadata sample_rate = idf.get_info('sample_rate') # Create the list of good digitizers and a digitizer to Antenna instance mapping. # These are: # toKeep -> mapping of digitizer number to array location # mapper -> mapping of Antenna instance to array location toKeep = [antennas[i].digitizer - 1 for i in good] mapper = [antennas[i] for i in good] # Create a list of unqiue stands to know what style of IDI file to create stands = set([antennas[i].stand.id for i in good]) # Figure out the output mode if os.path.splitext(filename)[1].find('.ms_') != -1: writer_class = measurementset.Ms else: if len(stands) > 255: writer_class = fitsidi.ExtendedIdi else: writer_class = fitsidi.Idi wallTime = time.time() readT, t, data = idf.read() setTime = t ref_time = t # Setup the set time as a python datetime instance so that it can be easily printed setDT = setTime.datetime print("Working on set #1 (%.3f seconds after set #1 = %s)" % ((setTime - ref_time), setDT.strftime("%Y/%m/%d %H:%M:%S.%f"))) # In order for the TBW stuff to actaully run, we need to run in with sub- # integrations. 8 sub-integrations (61.2 ms / 8 = 7.7 ms per section) # seems to work ok with a "reasonable" number of channels. nSec = 8 secSize = data.shape[1] // nSec # Loop over polarizations (there should be only 1) for pol in pols: print("-> %s" % pol) try: tempVis *= 0 # pylint:disable=undefined-variable except NameError: pass # Set up the progress bar so we can keep up with how the sub-integrations # are progressing pb = ProgressBar(max=nSec) sys.stdout.write(pb.show() + '\r') sys.stdout.flush() # Loop over sub-integrations (set by nSec) for k in range(nSec): blList, freq, vis = fxc.FXMaster(data[toKeep, k * secSize:(k + 1) * secSize], mapper, LFFT=LFFT, overlap=overlap, pfb=pfb, include_auto=True, verbose=False, sample_rate=sample_rate, central_freq=0.0, pol=pol, return_baselines=True, gain_correct=True) toUse = numpy.where((freq >= 5.0e6) & (freq <= 93.0e6)) toUse = toUse[0] try: tempVis += vis except NameError: tempVis = vis pb.inc(amount=1) sys.stdout.write(pb.show() + '\r') sys.stdout.flush() # Average the sub-integrations together vis = tempVis / float(nSec) # Set up the FITS IDI file is we need to if pol == pols[0]: pol1, pol2 = fxc.pol_to_pols(pol) fits = writer_class(filename, ref_time=ref_time) fits.set_stokes(pols) fits.set_frequency(freq[toUse]) fits.set_geometry(site, [a for a in mapper if a.pol == pol1]) # Add the visibilities fits.add_data_set(setTime, readT, blList, vis[:, toUse], pol=pol) sys.stdout.write(pb.show() + '\r') sys.stdout.write('\n') sys.stdout.flush() print("-> Cummulative Wall Time: %.3f s (%.3f s per integration)" % ((time.time() - wallTime), (time.time() - wallTime))) fits.write() fits.close() del (fits) del (data) del (vis) return True