def get_sample_offset( fullparsetname, file, antenna_set, max_sample_number, sample_calibrated=False, ref_station='', ref_time=None, ): #---------------------------------------------------- #Calibrate station times. f_clock_offset = float( md.getClockCorrectionParset(fullparsetname, file['STATION_NAME'][0], antennaset=antenna_set)) blocklen = file['BLOCKSIZE'] if not sample_calibrated: #First step: Rounding the sample_number used to the nearest whole block since second start, including CLOCK_OFFSET. sample_offset = cr.hArray(int, 1, max_sample_number) cr.hModulus(sample_offset, blocklen) sample_offset = cr.asval(blocklen - sample_offset + int(f_clock_offset / file['SAMPLE_INTERVAL'][0])) else: if not ref_time: if not ref_station: f0 = file else: f0 = cr.open(ref_station) print 'WARNING, using a ref_station assumes the dumps were within the same second, if not then give a ref_time.' f0_clock_offset = float( md.getClockCorrectionParset(fullparsetname, f0['STATION_NAME'][0], antennaset='HBA0')) t0 = max(f0['SAMPLE_NUMBER'] ) * f0['SAMPLE_INTERVAL'][0] + f0_clock_offset else: t0 = ref_time[1] t = max_sample_number + f_clock_offset sample_offset = [ int((t0 - t) / file['SAMPLE_INTERVAL'][0]), (t0 - t) / file['SAMPLE_INTERVAL'][0] - int( (t0 - t) / file['SAMPLE_INTERVAL'][0]) ] return sample_offset
def getRef_time(filelist, fullparsetname=''): ''' Gets the reference time for a list of tbb files (station which start observing last). ''' times = cr.hArray(float, len(filelist), 0.0) seconds = cr.hArray(float, len(filelist), 0.0) for i, file in enumerate(filelist): tbb = cr.open(file) if fullparsetname: f_clock_offset = float( md.getClockCorrectionParset(fullparsetname, tbb['STATION_NAME'][0], antennaset=tbb['ANTENNA_SET'])) else: f_clock_offset = float( md.getClockCorrection(tbb['STATION_NAME'][0], antennaset=tbb['ANTENNA_SET'])) time_tbb = cr.hArray(float, len(tbb['SAMPLE_NUMBER']), fill=tbb['TIME']) time_tbb -= min(tbb['TIME']) sample_tbb = cr.hArray(float, len(tbb['SAMPLE_NUMBER']), fill=tbb['SAMPLE_NUMBER']) sample_tbb *= tbb['SAMPLE_INTERVAL'][0] sample_tbb += time_tbb times[i] = max(sample_tbb) + f_clock_offset seconds[i] = min(tbb['TIME']) tbb.close() if verbose: print 'Reference time: ', max(times), 'from file ', filelist[ cr.hFindBetweenOrEqual(times, max(times), max(times))[0]] return [min(seconds), max(times)]
startblock = options.startblock nblocks = options.nblocks blocksize = options.blocksize #nfmin=None#options.nfmin #nfmax=None#options.nfmax #nfreq = blocksize /2+1 #nfmax-nfmin nfmin = options.nfmin nfmax = options.nfmax nfreq = nfmax - nfmin print nfreq # Select even antennae selection = range(1, 48, 2) # Open datafile object f = cr.open('~/RESEARCH/VHECR/Data/' + filenames[0]) #f=cr.open(filenames[0], blocksize) # Only single file mode supported #f["ANTENNA_SET"]='LBA_OUTER' # Set antenna selection f.setAntennaSelection(selection) if not options.ntimesteps: ntimesteps = f["MAXIMUM_READ_LENGTH"] / (nblocks * blocksize) else: ntimesteps = options.ntimesteps print "Number of time steps:", ntimesteps # Get frequencies frequencies = f.getFrequencies()
def add_stations(filenames, outbfdir, tbin=12, dm=0, incoherent=True, skip_stations=[]): """ Using pycrtools function addBeams to beamform across stations. Parameters ---------- files : list list of .beam station files tbin : int bin time dm : float dispersion measure (0 if the data is already dedispersed) incoherent : bool True if incoherent beam addition, False if coherent skip_stations : list list of stations to skip when beamforming across stations """ # Opening files #files_all = glob.glob(filenames) #files_all.sort() files = [] for f in filenames: if len(skip_stations) > 0: st = f.split('/')[-1].split('_')[1] if st in skip_stations: print("Skipping station", st) continue try: cr.open(f) files.append(f) except: print(f, "could not be opened") files.sort() print("Reading files", files) # Beamforming beams = cr.open(files) TAB, dynspec, cleandynspec = bt.addBeams(beams, dyncalc=True, tbin=tbin, dm=dm, clean=True, incoherent=incoherent) # Defining output names f = files[0].split('/')[-1].replace('.beam', '').split('_') obs = [l for l in f if 'L' in l][0] pol = [p for p in f if 'pol' in p][0] #hba = [h for h in f if 'HBA' in h][0] date = [d for d in f if 'D' in d and 'T' in d][0] if outbfdir[-1] != '/': outbfdir = outbfdir + '/' tabname = outbfdir + '{0}_{1}_TAB_{2}'.format(obs, date, pol) dsname = outbfdir + '{0}_{1}_dynspec_{2}'.format(obs, date, pol) cdsname = outbfdir + '{0}_{1}_cleandynspec_{2}'.format(obs, date, pol) print("Writing files:") print(tabname, '\n', dsname, '\n', cdsname) # Saving beamformed data TAB.write(tabname + '.beam') dynspec.write(dsname + '.beam') cleandynspec.write(cdsname + '.beam') npTAB = TAB.toNumpy() npdynspec = dynspec.toNumpy() npcleandynspec = cleandynspec.toNumpy() np.save(tabname, npTAB) np.save(dsname, npdynspec) np.save(cdsname, npcleandynspec)
if zoom: if pulse==1: nblocks=560 freq_size=1147+1#1228 incoherent_dynspec = cr.hArray(float,[freq_size, nblocks]) else: nblocks=640 freq_size=1638 incoherent_dynspec = cr.hArray(float,[freq_size, nblocks]) else: tbin=16 nblocks=12224 incoherent_dynspec = cr.hArray(float,[ 8193, nblocks / tbin]) for i,file in enumerate(filenames): beams = cr.open(file) print 'Working on station: '+ beams['STATION_NAME'][0] full_dynspec=cr.hArray(float,(beams['BEAM_SPECLEN'],full_blocks)) filename = os.path.join(file,extra) full_dynspec.readfilebinary(filename,0) if zoom: zoom_dynspec=bt.cutdynspec(full_dynspec,pulse=pulse) zoom_dynspec.par.station = beams['STATION_NAME'][0] incoherent_dynspec+=zoom_dynspec/6. # zoom_beam=bt.cutbeam(beam) # zoom_dynspec=bt.rawdyncalc(zoom_beam,axis_val=True) if verbose: super_plot(zoom_dynspec,pulse=pulse,zoom=zoom) pdb.set_trace() # Used for now to pause and manually save the images. else:
-------------------------------------------------------- #Example script (CR) import pycrtools as cr from pycrtools.tasks import findrfi import matplotlib.pyplot as plt datafile = cr.open('VHECR_example.h5') plt.ion() plt.figure() -------------------------------------------------------- #FindRFI rfi = cr.trun("FindRFI", f = datafile, plot_prefix ='',nofblocks=10,verbose=False,startblock=0,save_plots=True) if rfi: datafile['SELECTED_DIPOLES'] = rfi.good_antennas -------------------------------------------------------- #Imaging from pycrtools.tasks import imager output = 'out_cr.test.fits' cr.trun("Imager", data = datafile, intgrfreq = True, nblocks=1,ntimesteps=3,startblock=8,NAXIS1=155,NAXIS2=155,output=output,CDELT1=-0.5,CDELT2=0.5, rfi_remove=rfi.dirty_channels,CRVAL1=231.9,CRVAL2=63.1,CTYPE1='ALON-AZP',CTYPE2='ALAT-AZP', FREQMIN=0.3e8,FREQMAX=0.8e8) ----------
'L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1a' + extra + 'beam', 'L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1a' + extra + 'beam', 'L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1a' + extra + 'beam', 'L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1a' + extra + 'beam', 'L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1a' + extra + 'beam', 'L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1b' + extra + 'beam', 'L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1b' + extra + 'beam', 'L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1b' + extra + 'beam', 'L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1b' + extra + 'beam', 'L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1b' + extra + 'beam', 'L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1b' + extra + 'beam' ] #filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1a'+extra+'beam'] #filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1b'+extra+'beam'] f = cr.open([BEAMDATA + beams_dir + fname for fname in filenames]) f['CAL_DELAY'] = [ 0.0, 7.61718749999e-10, -4.98046875e-09, -8.0078125e-09, -5.0390625e-09, -4.27734375e-09, 2.92968750002e-10, 7.51953124997e-10, -5.419921875e-09, -8.408203125e-09, -5.205078125e-09, -5.400390625e-09 ] #wrt CS002a, , CC hFFTWResample , ears, new beams with modified rel.ant.pos. #f['CAL_DELAY']=[0.0,8.00781249999e-10,-4.98046875e-09,-8.046875e-09,-5.05859375e-09,-4.21875e-09,3.12499999998e-10,7.81250000002e-10,-5.4296875e-09,-8.41796875e-09 ,-5.1953125e-09 ,-5.3515625e-09] #f['CAL_DELAY']=[0.0,8.00781249999e-10,-4.98046875e-09,-8.046875e-09,-5.05859375e-09,-4.21875e-09] #f['CAL_DELAY']=[3.12499999998e-10,7.81250000002e-10,-5.4296875e-09,-8.41796875e-09 ,-5.1953125e-09 ,-5.3515625e-09] #f['CAL_DELAY']=[0.0,6.4453125e-10,-5.37109375e-09,-8.4375e-09,-5.2734375e-09,-4.98046875e-09] RA = 1 #PSR B0329+54 -- Pulsar 0.71452 sec if RA:
'''Script to test the cross correlation of pulses. ''' import pycrtools as cr import numpy as np import sys; import os import pdb;# pdb.set_trace() import Beam_Tools as bt filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1.multi_station.beam'] filenames = [filenames[0],filenames[0]] #filenames = filenames[0:7:5] beams = cr.open(filenames) beams['DM'] = 26.76 block_range=np.arange(240,320) nblocks = len(block_range) blocklen = 16384 speclen = 8193 # Do mean of complex or timeseries mean_fft =1 #pdb.set_trace() if mean_fft: cross_correlation = cr.hArray(float,[1,16384]) #beams.empty('TIMESERIES_DATA')[0] fft_matrix = cr.hArray(complex,[nblocks,speclen]) # ,len(filenames)
'L43784_D20120125T211154.887Z_CS002_R000_tbb.pol0b' + extra[set] + 'beam', 'L43784_D20120125T211154.867Z_CS003_R000_tbb.pol0b' + extra[set] + 'beam', 'L43784_D20120125T211154.887Z_CS004_R000_tbb.pol0b' + extra[set] + 'beam', 'L43784_D20120125T211154.866Z_CS005_R000_tbb.pol0b' + extra[set] + 'beam', 'L43784_D20120125T211154.871Z_CS006_R000_tbb.pol0b' + extra[set] + 'beam', 'L43784_D20120125T211154.887Z_CS007_R000_tbb.pol0b' + extra[set] + 'beam' ], } #beams_suffix = '*pol0?'+extra[set]+'beam' #filenames = glob.glob(BEAMDATA+beams_dir+beams_suffix) beams = cr.open([BEAMDATA + beams_dir + file for file in filenames[set]]) beams['NCHUNKS'] = 191 #beams['CAL_DELAY']=[-5.4375e-09,-5.96875e-09,0.0,-2.9375e-09,9.375e-11,2.8125e-10] # wrt CS007 #beams['CAL_DELAY']=[5.4375e-09,5.96875e-09,0.0,2.9375e-09,-9.375e-11,-2.8125e-10] # wrt CS007 , negative #beams['CAL_DELAY']=[0.0,6.875e-10,-5.4375e-09,-8.3125e-09,1.0625e-09,-4.9375e-09] # wrt CS002 , #beams['CAL_DELAY']=[0.0,-6.875e-10,5.4375e-09,8.3125e-09,-1.0625e-09,4.9375e-09] # wrt CS002 , negative #beams['CAL_DELAY']=[0.0,-3.4375e-09,6.25000000003e-10,9.37500000001e-10,9.37500000001e-10,-1.84375e-08] # wrt CS002 , CC values #beams['CAL_DELAY']=[0.0,-2.5e-09,-3.34375e-09,-3.5625e-09,-3.6875e-09,-1.721875e-08] # wrt CS002 , abs(CC) hFFTWResample #beams['CAL_DELAY']=[0.0,2.5e-09,3.34375e-09,3.5625e-09,3.6875e-09,1.721875e-08] # wrt CS002 , abs(CC) hFFTWResample , negative #beams['CAL_DELAY']=[0.0,1.6875e-09,2.375e-09,-5.84375e-09,2.59375e-09,-1.3375e-08] # wrt CS002 , CC hFFTWResample #beams['CAL_DELAY']=[0.0,1.6859375e-09,2.3625e-09,-5.8421875e-09,2.6046875e-09,-1.3371875e-08] # wrt CS002 , CC hFFTWResample, more acurate #beams['CAL_DELAY']=[0.0,-1.6875e-09,-2.375e-09,5.84375e-09,-2.59375e-09,1.3375e-08] # wrt CS002 , CC hFFTWResample , negative #beams['CAL_DELAY']/=2.68 # Magic value :) #beams['CAL_DELAY']=[0.0,6.4453125e-10,-5.37109375e-09,-8.4375e-09,-5.2734375e-09,-4.98046875e-09 ] #wrt CS002 , CC hFFTWResample , upsampling #beams['CAL_DELAY']*=-1
#Adding if statement, in case no need to do the "beams" and only the imaging part, ugly fix for now. #---------------------------------------------------- #Hard coded stuff that will go filename = '/vol/astro/lofar/frats/tbb/data/L43784_ev1/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5' polarization = 0 blocklen = 2**14 dm = 26.76 freq_range = [166, 168] #---------------------------------------------------- #File selection #Open given TBB file. fname = filename # args[0] file = cr.open(fname) #---------------------------------------------------- #General parameters. FRATS_ANALYSIS = os.environ["FRATS_ANALYSIS"].rstrip('/') + '/' Obs_ID = fname.split('/')[-1].split('_')[0] outdir = FRATS_ANALYSIS + Obs_ID + '/all_sky/' # Create output directory, if not already present. if not os.path.isdir(outdir) or not os.path.isdir(outdir + '/beams/'): os.makedirs(outdir + '/beams/') if not os.path.isdir(outdir + '/RFI/'): os.mkdir(outdir + '/RFI/')
'BeamFormer' ) #Using this since it seems the program runs faster when the task is preloaded (if other task is "loaded" then it runs slower). cr.treset() if altair: # f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5') # filefilter = '/data/FRATS/data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5' # f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5') # filefilter = '/data/FRATS/data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5' # f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.867Z_CS003_R000_tbb.h5') # filefilter = '/data/FRATS/data/L43784_D20120125T211154.867Z_CS003_R000_tbb.h5' # f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.887Z_CS004_R000_tbb.h5') # filefilter = '/data/FRATS/data/L43784_D20120125T211154.887Z_CS004_R000_tbb.h5' # f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.866Z_CS005_R000_tbb.h5') # filefilter = '/data/FRATS/data/L43784_D20120125T211154.866Z_CS005_R000_tbb.h5' f1 = cr.open( '/data/FRATS/data/L43784_D20120125T211154.887Z_CS007_R000_tbb.h5') filefilter = '/data/FRATS/data/L43784_D20120125T211154.887Z_CS007_R000_tbb.h5' output_dir = '/Users/eenriquez/RESEARCH/Pulsars/Results/' f2 = cr.open( '/data/FRATS/data/L43784_D20120125T211154.887Z_CS007_R000_tbb.h5') else: f1 = cr.open( '~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5' ) filefilter = '~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5' # f1=cr.open('~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5') # filefilter = '~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5' output_dir = '~/RESEARCH/Pulsars/Results/' f2 = cr.open( '~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5'
filenames = glob.glob(LOFARDATA + '*.h5') filter = 'HBA0' diff = 0 Two_d = 1 if Two_d: filenames = [filenames[0]] xpos1 = cr.hArray(float, (len(filenames), 96)) ypos1 = cr.hArray(float, (len(filenames), 96)) zpos1 = cr.hArray(float, (len(filenames), 96)) stations = [] for i, file in enumerate(filenames): f = cr.open(file) f['ANTENNA_SET'] = filter if diff: antpos = f.getRelativeAntennaPositions( ) - md.getRelativeAntennaPositionsNew(f['STATION_NAME'][0], filter) else: antpos = md.getRelativeAntennaPositionsNew(f['STATION_NAME'][0], filter) #X-axis positions xpos = cr.hArray(float, [96]) xpos[...].copy(antpos[..., 0]) xpos1[i] = xpos #Y-axis positions
Obs_ID ='L74100' polarization=0 FRATS_ANALYSIS=os.environ["FRATS_ANALYSIS"].rstrip('/')+'/' ##beam_suffix='*pol%i_*HBA?.beam'%(polarization) #beam_suffix='*pol0_sample*' beam_suffix='*CS00[2-7]*pol0_sample*' filenames=glob.glob(FRATS_ANALYSIS+Obs_ID+'_new'+'/beam.results/'+beam_suffix) #filenames = ['L74100_D20121107T191144.924Z_CS005_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.993Z_CS030_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.993Z_CS030_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.929Z_CS011_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.916Z_CS004_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.929Z_CS011_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.964Z_CS002_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.924Z_CS005_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.964Z_CS002_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.916Z_CS004_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam'] #filenames = ['L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.993Z_CS030_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam','L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam','L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam','L74100_D20121107T191144.964Z_CS002_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam','L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam'] print 'Using files for imaging: ',filenames #f = cr.open([FRATS_ANALYSIS+Obs_ID+'_new'+'/beam.results/'+filename for filename in filenames]) f = cr.open(filenames) #PSR B0329+54 -- Pulsar 0.71452 sec alpha = hms2deg(03,32,59.37); # Right assention delta = dms2deg(54,34,44.9); # Declination ctype1 ='RA---SIN' ctype2 ='DEC--SIN' f['DM'] = 26.76 #f['CAL_DELAY']=cr.hArray(float, [8], fill=[1.00586e-09,6.32813e-09,-7.97852e-09,9.86328e-09,5.83984e-09,0,-4.26758e-09,3.55469e-09]) # len=24 slice=[0:24]) ST_DELAYS = bt.ccBeams(f,freq_range=[175,195],time_range=[0.346,0.3495],verbose=1) f['CAL_DELAY']=ST_DELAYS output = FRATS_ANALYSIS+Obs_ID+'_new'+'/L74100.cal_on.fits'
#! /usr/bin/env python import pycrtools as cr import os import pdb;# pdb.set_trace() LOFARDATA=os.environ["LOFARDATA"].rstrip('/')+'/' filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1.multi_station.beam'] f = cr.open([LOFARDATA+fname for fname in filenames]) f['CAL_DELAY']=[0.0,1.00e-12] f['DM'] = 26.76 output = 'out_pul.sub_st.high_res.fits' pdb.set_trace() cr.trun("Imager", data = f, intgrfreq = False, nblocks=30,ntimesteps=10,startblock=100,NAXIS1=33,NAXIS2=33,FREQMIN=1.5e8,FREQMAX=1.69e8,CDELT1=-.125,CDELT2=.125,CTYPE1 ='ALON_SIN',CTYPE2 ='ALAT_SIN',output=output)
"""Run beamformer on Pulsar: /vol/astro/lofar/frats/tbb/data/. """ import sys import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import pycrtools as cr from pytmf import * from pycrtools import tools from pycrtools.tasks import beamformer #---------------------------------------------------- fname = sys.argv[1] f = cr.open(fname) utc = tools.strdate2jd(f['TIME_HR'][0]) #---------------------------------------------------- outdir = '../beam.results/beams.sub_station/' substation = 1 inner = 0 detail_name = '.pol0a.new.multi_station' detail_name2 = '.pol0b.new.multi_station' #---------------------------------------------------- #Pointing and LOFAR posisition info. alpha = hms2rad(3, 32, 59.37) # Right assention delta = dms2rad(54, 34, 44.9) # Declination phi = deg2rad(52.915122495) #(LOFAR Superterp)
def dyncalc_multibeam(filename=None, beams=None, nbeam=0, save_file=False, from_file=False, fraction=None, tbin=1, clean=False): ''' Calculates the dynamic spectrum. =============== ===== =================================================================== *beams* None Input array. *nbeam* 0 Beam to work with, if ()beams has stored multiple ones. *save_file* False Saves a file in hArray format with additional information besides the array values. *from_file* False Read cleandynspec from file. *fraction* None If not None, then a list of the form [x,y] such that extracting the fraction x/y of the data. with x>-1, and y>=x. *tbin* 1 If >1 integrates over this number of blocks. Or time binning. *clean* False If True it calculates the cleaned spectrum. =============== ===== =================================================================== Example:: import Beam_Tools as bt dynspec = bt.dyncalc(filename) or:: import Beam_Tools as bt dynspec,cleandynspec = bt.dyncalc(beams,tbin=16,clean=True,save_file=True) The regular and clean dynamic spectra (all blocks) are returned and stored in ``Task.dynspec`` and ``Task.cleandynspec`` respectively. ''' raise NotImplementedError if beams == None and filename == None: raise ValueError( 'Need to provide either the filename or the opened .beam file') if nbeam != 0 or from_file or save_file: raise KeyError("Keyword is invalid for now: " + key) t0 = time.clock() if beams == None: beams = cr.open(filename) elif filename == None: filename = beams['FILENAMES'][0] # filename_bin = os.path.join(filename,"data.bin") #Maybe needed in the future if using "from_file" option. speclen = beams['BLOCKSIZE'] / 2 + 1 block_duration = beams['BLOCKSIZE'] * beams['SAMPLE_INTERVAL'][0] nblocks = beams['NCHUNKS'] * beams['BEAM_NBLOCKS'] if not fraction: fraction = [1, 1] else: if type(fraction) != type([]) or len(fraction) != 2 or fraction[ 0] > fraction[1] or fraction[0] < 0 or fraction[1] < 0: raise ValueError( 'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.' ) if fraction[0] == 0: fraction[0] = 1 nblocks = int(nblocks / fraction[1]) start_time = (fraction[0] - 1) * (block_duration * nblocks) / fraction[1] end_time = fraction[0] * (block_duration * nblocks) / fraction[1] beam = beams.empty('FFT_DATA') tm = cr.hArray(float, beam) dynspec = cr.hArray(float, [nblocks, beam.shape()[0], speclen]) block_range = range((fraction[0] - 1) * nblocks, (fraction[0]) * nblocks) pdb.set_trace() #Reading beams. for block in block_range: print ' Calculation at {0:.2%} \r'.format( float(block) / len(block_range)), sys.stdout.flush() beams.getFFTData(beam, block) tm[...].spectralpower2(beam[...]) dynspec[block] = tm #Time integration. if tbin > 1: dynspec = cr.hArray_toNumpy(dynspec) dynspec = dynspec.reshape((dynspec.shape[0] / tbin, tbin, dynspec.shape[1], dynspec.shape[2])) dynspec = np.sum(dynspec, axis=1) dynspec = cr.hArray(dynspec) #Rearranging dimensions. dynspec = cr.hArray_toNumpy(dynspec) # dynspec = np.rollaxis(dynspec,0,3) dynspec = np.swapaxes(dynspec, 0, 1) dynspec = np.swapaxes(dynspec, 1, 2) dynspec = cr.hArray( dynspec.copy()) #The .copy() is quick&dirty way to avoid a bug. #Cleaning dynamic spectrum. if clean: avspec = cr.hArray(float, speclen, fill=0.0) cleandynspec = cr.hArray(float, dynspec, fill=0.0) for dim in range(dynspec.shape()[0]): single_dynspec = cr.hArray(float, dynspec.shape()[1:], dynspec[dim]) single_dynspec[...].addto(avspec) cleandynspec[dim] = single_dynspec / avspec #Create a frequency vector frequencies = beams['BEAM_FREQUENCIES'] #Create a time vector times = cr.hArray( float, int(round((end_time - start_time) / (block_duration * tbin))), name="Time", units=("", "s")) times.fillrange(start_time, block_duration * tbin) #Adding parameters dynspec.par.yvalues = frequencies dynspec.par.xvalues = times dynspec.par.tbin = tbin if clean: cleandynspec.par.yvalues = frequencies cleandynspec.par.xvalues = times cleandynspec.par.tbin = tbin #Saving file(s). if save_file: dynspec.write(os.path.join(filename, "dynspec"), nblocks=1, block=0, clearfile=True) print 'Saving binary in %s' % os.path.join(filename, "dynspec.pcr") if clean: cleandynspec.write(os.path.join(filename, "clean_dynspec"), nblocks=1, block=0, clearfile=True) print 'Saving binary in %s' % os.path.join(filename, "clean_dynspec.pcr") print "Finished - total time used:", time.clock() - t0, "s." if clean: return dynspec, cleandynspec else: return dynspec
blocksize = int(wide * 1024) files = args count = 0 for n in files: count += 1 data = [array] * count shiftmax = 0 f = [array] * count #-------------read in RCUs--------------------------------------- for i in range(0, count, 1): data[i] = cr.open(files[i], blocksize) f[i] = cr.TBBData(files[i]) if i == 0: rcun = data[i]["DIPOLE_NAMES"] nant = len(rcun) shifts = zeros((count, nant), dtype=int) if i != 0: rcun2 = data[i]["DIPOLE_NAMES"] rcun = hstack((rcun, rcun2)) del rcun2 #-------------read in antenna positions ------------------------- for i in range(0, count, 1):
beam_suffix) beam_suffix2 = '*CS00[2-7]*pol%i?.new.beam' % (polarization) filenames2 = glob.glob( FRATS_ANALYSIS + '../testing/PSR_B0329.54/beam.results/beams.sub_station/' + beam_suffix2) else: beam_suffix = '*pol%i.*HBA?.LOFAR_centered.beam' % (polarization) filenames = glob.glob(FRATS_ANALYSIS + Obs_ID2 + '/beam.results/' + beam_suffix) beam_suffix2 = '*pol%i.*HBA?.beam' % (polarization) filenames2 = glob.glob(FRATS_ANALYSIS + Obs_ID2 + '/beam.results/' + beam_suffix2) print 'Using files for calibrating: ', filenames beams = cr.open(sorted(filenames)) print 'Using files for imaging: ', filenames2 beams2 = cr.open(sorted(filenames2)) #---------------------- #Calibration #PSR B0329+54 -- Pulsar 0.71452 sec alpha = hms2deg(03, 32, 59.37) # Right assention delta = dms2deg(54, 34, 44.9) # Declination ctype1 = 'RA---SIN' ctype2 = 'DEC--SIN'
def get_noise(event_id): nstations=0 try: #for u in np.arange(1): event = crdb.Event(db=db, id=event_id) event_time=np.asarray(event["lora_time"]) event_time=event_time[event_time>1.0] event_time=np.min(event_time) #if event_time<100.0: # break print(event_time) time = Time(event_time, format='unix', scale='utc',location=loc) time.delta_ut1_utc = 0. LST=time.sidereal_time('apparent').hour print('event time UTC: {0}'.format(event_time)) print('event time LST: {0}'.format(LST)) stations = [] #collect stations with "GOOD" status for event for f in event.datafiles: stn=[] stn.extend(f.stations) #print stn[0].stationname #print f.stations.stationname if stn[0].stationname == "CS001" or stn[0].stationname == "CS002" or stn[0].stationname == "CS003" or stn[0].stationname == "CS004" or stn[0].stationname == "CS005" or stn[0].stationname == "CS006" or stn[0].stationname == "CS007" or stn[0].stationname == "CS011" or stn[0].stationname == "CS013" or stn[0].stationname == "CS017" or stn[0].stationname == "CS021" or stn[0].stationname == "CS026" or stn[0].stationname == "CS028" or stn[0].stationname == "CS030" or stn[0].stationname == "CS031" or stn[0].stationname == "CS032" or stn[0].stationname == "CS101" or stn[0].stationname == "CS103" or stn[0].stationname == "CS301" or stn[0].stationname == "CS302" or stn[0].stationname == "CS401" or stn[0].stationname == "CS501": if stn[0].status=="GOOD": stations.extend(f.stations) nstations=len(stations) except: print('no event at this point') for s in np.arange(nstations): station_flag=0 station=stations[s] # The following steps are copied from cr_physics pipeline # there are a million try/excepts because I ran into lots of specific errors I didn't want to handle try: # Open file f = cr.open(station.datafile.settings.datapath + '/' + station.datafile.filename) antenna_set= f["ANTENNA_SET"] # Check if we are dealing with LBA or HBA observations if "LBA" in f["ANTENNA_SET"]: print ('LBA event') else: print ('HBA event') continue except: print ('no event at antennas') continue # Read LORA information try: tbb_time = f["TIME"][0] max_sample_number = max(f["SAMPLE_NUMBER"]) min_sample_number = min(f["SAMPLE_NUMBER"]) (tbb_time_sec, tbb_time_nsec) = lora.nsecFromSec(tbb_time, logfile=os.path.join(lora_directory,lora_logfile)) (block_number_lora, sample_number_lora) = lora.loraTimestampToBlocknumber(tbb_time_sec, tbb_time_nsec, tbb_time, max_sample_number, blocksize=blocksize) except: continue # Check if starting time in sample units (SAMPLE_NUMBER) does not deviate among antennas try: sample_number_per_antenna = np.array(f["SAMPLE_NUMBER"]) median_sample_number = np.median(sample_number_per_antenna) data_length = np.median(np.array(f["DATA_LENGTH"])) deviating_antennas = np.where( np.abs(sample_number_per_antenna - median_sample_number) > data_length/4)[0] nof_deviating_antennas = len(deviating_antennas) print ('Number of deviating antennas: %d' % nof_deviating_antennas) except: continue try: frequencies = f["FREQUENCY_DATA"] print ('blocksize: {0}'.format(f["BLOCKSIZE"])) # Get bandpass filter nf = f["BLOCKSIZE"] / 2 + 1 ne = int(10. * nf / f["CLOCK_FREQUENCY"]) bandpass_filter.fill(0.) bandpass_filter[int(nf * 30.0 / 100.)-(ne/2):int(nf * 80.0 / 100.)+(ne/2)] = 1.0 gaussian_weights = cr.hArray(cr.hGaussianWeights(ne, 4.0)) cr.hRunningAverage(bandpass_filter, gaussian_weights) except: continue try: raw_data = f["TIMESERIES_DATA"].toNumpy() # Find outliers tmp = np.max(np.abs(raw_data), axis=1) outlier_antennas = np.argwhere(np.abs(tmp-np.median(tmp[tmp>0.1])) > 2*np.std(tmp[tmp>0.1])).ravel() print("Outlier antennas", outlier_antennas) except: print 'no raw data' continue try: # Get calibration delays to flag antennas with wrong calibration values try: cabledelays = cr.hArray(f["DIPOLE_CALIBRATION_DELAY"]) cabledelays = np.abs(cabledelays.toNumpy()) except: print 'problem with cable delays' continue # Find RFI and bad antennas findrfi = cr.trun("FindRFI", f=f, nofblocks=10, plotlist=[], apply_hanning_window=True, hanning_fraction=0.2, bandpass_filter=bandpass_filter) print "Bad antennas", findrfi.bad_antennas antenna_ids_findrfi = f["SELECTED_DIPOLES"] nAnt=len(f["SELECTED_DIPOLES"]) bad_antennas_spikes = [] bad_antennas = findrfi.bad_antennas[:] dipole_names = f["SELECTED_DIPOLES"] good_antennas = [n for n in dipole_names if n not in bad_antennas] station["crp_bad_antennas_power"] = findrfi.bad_antennas station["crp_bad_antennas_spikes"] = bad_antennas_spikes selected_dipoles = [] for i in range(len(dipole_names) / 2): if dipole_names[2 * i] in good_antennas and dipole_names[2 * i + 1] in good_antennas and f.nof_consecutive_zeros[2 * i] < 512 and f.nof_consecutive_zeros[2 * i + 1] < 512 and cabledelays[2 * i] < 150.e-9 and cabledelays[2 * i + 1] < 150.e-9: selected_dipoles.extend([dipole_names[2 * i], dipole_names[2 * i + 1]]) f["SELECTED_DIPOLES"] = selected_dipoles station["crp_selected_dipoles"] = selected_dipoles nDipoles=len(selected_dipoles) except: print 'issue with RFI' continue try: print block_number_lora nF= len(frequencies.toNumpy()) all_ffts=np.zeros([nAvg,nDipoles,nF]) all_ffts_cleaned=np.zeros([nAvg,nDipoles,nF]) except: continue #______________________________________________________________________ block_number=0 for i in np.arange(nAvg): try: # make sure not to include the signal window in the average if abs(block_number-block_number_lora)<5: block_number=block_number+10 fft_data = f.empty("FFT_DATA") #f.getFFTData(fft_data, block_number_lora, True, hanning_fraction=0.2, datacheck=True) # this is what is in the pipeline f.getFFTData(fft_data, block_number, True, hanning_fraction=0.2, datacheck=True) # Apply bandpass fft_data[...].mul(bandpass_filter) # Normalize spectrum fft_data /= f["BLOCKSIZE"] fft_hold=fft_data # Reject DC component fft_data[..., 0] = 0.0 # Also reject 1st harmonic (gives a lot of spurious power with Hanning window) fft_data[..., 1] = 0.0 # Flag dirty channels (from RFI excission) fft_data[..., cr.hArray(findrfi.dirty_channels)] = 0 # factor of two because reall FFT all_ffts[i]=2*np.abs(fft_hold.toNumpy())**2 all_ffts_cleaned[i]=2*np.abs(fft_data.toNumpy())**2 badFreq=frequencies.toNumpy()[findrfi.dirty_channels]/1e6 nBadChannelsFilt=len(badFreq[(badFreq>=30.0)*(badFreq<=80.0)]) if nBadChannelsFilt>1: print 'n bad channels: {0}'.format(nBadChannelsFilt) continue block_number=block_number+1 except: print 'error' continue try: #for y in np.arange(1): fft_avg=np.average(all_ffts_cleaned,axis=0) freq=frequencies.toNumpy() df=(freq[1]-freq[0])/1e6 freq_new=np.arange(30,81,1) fft_resample=np.zeros([nAnt,nResample]) except: print 'error in average' continue for n in np.arange(nDipoles): try: fft_use=fft_avg[n][fft_avg[n]>1e-100] freq_use=freq[fft_avg[n]>1e-100] if len(fft_avg[n])>len(fft_use): station_flag=1 f=interp1d(freq_use/1e6,fft_use) f_new=f(freq_new) start_f=np.argmin(np.abs((freq/1e6)-30)) stop_f=np.argmin(np.abs((freq/1e6)-80)) fft_resample[n]=f_new*(1/df) except: station_flag=1 print 'issue with interp' analysisinfo={'event_number': event_id,'station': station.stationname,'UTC_time':event_time,'LST':LST,'frequencies':freq,'FFT_data':fft_avg,'frequencies_50':freq_new,'FFT_data_resampled':fft_resample,'flag': station_flag,'antenna_set':antenna_set,'selected_dipoles': dipole_names,'bad_dipoles':bad_antennas,'nBadChannelsFilt':nBadChannelsFilt} outputfile=open(station.stationname+'/'+str(int(event_id))+'_noise_OUTER.p','w') pickle.dump(analysisinfo,outputfile) outputfile.close() print '{0} done'.format(station.stationname) print 'done with event'
#! /usr/bin/env python '''Images from tbbs. ''' import pycrtools as cr from pytmf import * import os import pdb;# pdb.set_trace() LOFARDATA=os.environ["LOFARDATA"].rstrip('/')+'/' BEAMOUT=os.environ["BEAMOUT"].rstrip('/')+'/' event_dir ='L43784_ev1/' filename = 'L43784_D20120125T211154.887Z_CS002_R000_tbb.h5' f = cr.open(LOFARDATA+event_dir+filename) RA = 2 #---------------------------------------------------- #RFI excission. # Find RFI and bad antennas rfi = cr.trun("FindRFI", f=f,nofblocks=100,verbose=False,startblock=0,save_plots=False) #print rfi.dirty_channels #print rfi.good_antennas if rfi: f['SELECTED_DIPOLES'] = rfi.good_antennas #PSR B0329+54 -- Pulsar 0.71452 sec
inner_tiles = options.inner_tiles polarization = options.polarization station_centered = options.station_centered blocklen = options.blocklen flag_antenna = options.flag_antenna rfi_find = options.rfi_find default_input = options.default_input ref_station = options.ref_station ref_time = options.ref_time obs_mode = options.obs_mode parset_time = options.parset_time #---------------------------------------------------- #Open given TBB file. fname = args[0] file = cr.open(fname) #---------------------------------------------------- #General parameters logistics. FRATS_ANALYSIS = os.environ["FRATS_ANALYSIS"].rstrip('/') + '/' Obs_ID = fname.split('/')[-1].split('_')[0] Obs_ID2 = fname.split('/')[-1].split('.')[0][:-2] # Minute accuracy. outdir = FRATS_ANALYSIS + Obs_ID2 # Create output directories, if not already present. if not os.path.isdir(outdir) or not os.path.isdir(outdir + '/beam.results/'): os.makedirs(outdir + '/beam.results/') if not os.path.isdir(outdir + '/RFI/'): os.mkdir(outdir + '/RFI/')