def get_sample_offset(
fullparsetname,
file,
antenna_set,
max_sample_number,
sample_calibrated=False,
ref_station='',
ref_time=None,
):
#----------------------------------------------------
#Calibrate station times.
f_clock_offset = float(
md.getClockCorrectionParset(fullparsetname,
file['STATION_NAME'][0],
antennaset=antenna_set))
blocklen = file['BLOCKSIZE']
if not sample_calibrated:
#First step: Rounding the sample_number used to the nearest whole block since second start, including CLOCK_OFFSET.
sample_offset = cr.hArray(int, 1, max_sample_number)
cr.hModulus(sample_offset, blocklen)
sample_offset = cr.asval(blocklen - sample_offset +
int(f_clock_offset /
file['SAMPLE_INTERVAL'][0]))
else:
if not ref_time:
if not ref_station:
f0 = file
else:
f0 = cr.open(ref_station)
print 'WARNING, using a ref_station assumes the dumps were within the same second, if not then give a ref_time.'
f0_clock_offset = float(
md.getClockCorrectionParset(fullparsetname,
f0['STATION_NAME'][0],
antennaset='HBA0'))
t0 = max(f0['SAMPLE_NUMBER']
) * f0['SAMPLE_INTERVAL'][0] + f0_clock_offset
else:
t0 = ref_time[1]
t = max_sample_number + f_clock_offset
sample_offset = [
int((t0 - t) / file['SAMPLE_INTERVAL'][0]),
(t0 - t) / file['SAMPLE_INTERVAL'][0] - int(
(t0 - t) / file['SAMPLE_INTERVAL'][0])
]
return sample_offset
def getRef_time(filelist, fullparsetname=''):
''' Gets the reference time for a list of tbb files (station which start observing last).
'''
times = cr.hArray(float, len(filelist), 0.0)
seconds = cr.hArray(float, len(filelist), 0.0)
for i, file in enumerate(filelist):
tbb = cr.open(file)
if fullparsetname:
f_clock_offset = float(
md.getClockCorrectionParset(fullparsetname,
tbb['STATION_NAME'][0],
antennaset=tbb['ANTENNA_SET']))
else:
f_clock_offset = float(
md.getClockCorrection(tbb['STATION_NAME'][0],
antennaset=tbb['ANTENNA_SET']))
time_tbb = cr.hArray(float,
len(tbb['SAMPLE_NUMBER']),
fill=tbb['TIME'])
time_tbb -= min(tbb['TIME'])
sample_tbb = cr.hArray(float,
len(tbb['SAMPLE_NUMBER']),
fill=tbb['SAMPLE_NUMBER'])
sample_tbb *= tbb['SAMPLE_INTERVAL'][0]
sample_tbb += time_tbb
times[i] = max(sample_tbb) + f_clock_offset
seconds[i] = min(tbb['TIME'])
tbb.close()
if verbose:
print 'Reference time: ', max(times), 'from file ', filelist[
cr.hFindBetweenOrEqual(times, max(times), max(times))[0]]
return [min(seconds), max(times)]
startblock = options.startblock
nblocks = options.nblocks
blocksize = options.blocksize
#nfmin=None#options.nfmin
#nfmax=None#options.nfmax
#nfreq = blocksize /2+1 #nfmax-nfmin
nfmin = options.nfmin
nfmax = options.nfmax
nfreq = nfmax - nfmin
print nfreq
# Select even antennae
selection = range(1, 48, 2)
# Open datafile object
f = cr.open('~/RESEARCH/VHECR/Data/' + filenames[0])
#f=cr.open(filenames[0], blocksize) # Only single file mode supported
#f["ANTENNA_SET"]='LBA_OUTER'
# Set antenna selection
f.setAntennaSelection(selection)
if not options.ntimesteps:
ntimesteps = f["MAXIMUM_READ_LENGTH"] / (nblocks * blocksize)
else:
ntimesteps = options.ntimesteps
print "Number of time steps:", ntimesteps
# Get frequencies
frequencies = f.getFrequencies()
def add_stations(filenames,
outbfdir,
tbin=12,
dm=0,
incoherent=True,
skip_stations=[]):
"""
Using pycrtools function addBeams to beamform across stations.
Parameters
----------
files : list
list of .beam station files
tbin : int
bin time
dm : float
dispersion measure (0 if the data is already dedispersed)
incoherent : bool
True if incoherent beam addition, False if coherent
skip_stations : list
list of stations to skip when beamforming across stations
"""
# Opening files
#files_all = glob.glob(filenames)
#files_all.sort()
files = []
for f in filenames:
if len(skip_stations) > 0:
st = f.split('/')[-1].split('_')[1]
if st in skip_stations:
print("Skipping station", st)
continue
try:
cr.open(f)
files.append(f)
except:
print(f, "could not be opened")
files.sort()
print("Reading files", files)
# Beamforming
beams = cr.open(files)
TAB, dynspec, cleandynspec = bt.addBeams(beams,
dyncalc=True,
tbin=tbin,
dm=dm,
clean=True,
incoherent=incoherent)
# Defining output names
f = files[0].split('/')[-1].replace('.beam', '').split('_')
obs = [l for l in f if 'L' in l][0]
pol = [p for p in f if 'pol' in p][0]
#hba = [h for h in f if 'HBA' in h][0]
date = [d for d in f if 'D' in d and 'T' in d][0]
if outbfdir[-1] != '/': outbfdir = outbfdir + '/'
tabname = outbfdir + '{0}_{1}_TAB_{2}'.format(obs, date, pol)
dsname = outbfdir + '{0}_{1}_dynspec_{2}'.format(obs, date, pol)
cdsname = outbfdir + '{0}_{1}_cleandynspec_{2}'.format(obs, date, pol)
print("Writing files:")
print(tabname, '\n', dsname, '\n', cdsname)
# Saving beamformed data
TAB.write(tabname + '.beam')
dynspec.write(dsname + '.beam')
cleandynspec.write(cdsname + '.beam')
npTAB = TAB.toNumpy()
npdynspec = dynspec.toNumpy()
npcleandynspec = cleandynspec.toNumpy()
np.save(tabname, npTAB)
np.save(dsname, npdynspec)
np.save(cdsname, npcleandynspec)
if zoom:
if pulse==1:
nblocks=560
freq_size=1147+1#1228
incoherent_dynspec = cr.hArray(float,[freq_size, nblocks])
else:
nblocks=640
freq_size=1638
incoherent_dynspec = cr.hArray(float,[freq_size, nblocks])
else:
tbin=16
nblocks=12224
incoherent_dynspec = cr.hArray(float,[ 8193, nblocks / tbin])
for i,file in enumerate(filenames):
beams = cr.open(file)
print 'Working on station: '+ beams['STATION_NAME'][0]
full_dynspec=cr.hArray(float,(beams['BEAM_SPECLEN'],full_blocks))
filename = os.path.join(file,extra)
full_dynspec.readfilebinary(filename,0)
if zoom:
zoom_dynspec=bt.cutdynspec(full_dynspec,pulse=pulse)
zoom_dynspec.par.station = beams['STATION_NAME'][0]
incoherent_dynspec+=zoom_dynspec/6.
# zoom_beam=bt.cutbeam(beam)
# zoom_dynspec=bt.rawdyncalc(zoom_beam,axis_val=True)
if verbose:
super_plot(zoom_dynspec,pulse=pulse,zoom=zoom)
pdb.set_trace() # Used for now to pause and manually save the images.
else:
--------------------------------------------------------
#Example script (CR)
import pycrtools as cr
from pycrtools.tasks import findrfi
import matplotlib.pyplot as plt
datafile = cr.open('VHECR_example.h5')
plt.ion()
plt.figure()
--------------------------------------------------------
#FindRFI
rfi = cr.trun("FindRFI", f = datafile, plot_prefix ='',nofblocks=10,verbose=False,startblock=0,save_plots=True)
if rfi:
datafile['SELECTED_DIPOLES'] = rfi.good_antennas
--------------------------------------------------------
#Imaging
from pycrtools.tasks import imager
output = 'out_cr.test.fits'
cr.trun("Imager", data = datafile, intgrfreq = True, nblocks=1,ntimesteps=3,startblock=8,NAXIS1=155,NAXIS2=155,output=output,CDELT1=-0.5,CDELT2=0.5, rfi_remove=rfi.dirty_channels,CRVAL1=231.9,CRVAL2=63.1,CTYPE1='ALON-AZP',CTYPE2='ALAT-AZP', FREQMIN=0.3e8,FREQMAX=0.8e8)
----------
'L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1a' + extra + 'beam',
'L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1a' + extra + 'beam',
'L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1a' + extra + 'beam',
'L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1a' + extra + 'beam',
'L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1a' + extra + 'beam',
'L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1b' + extra + 'beam',
'L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1b' + extra + 'beam',
'L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1b' + extra + 'beam',
'L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1b' + extra + 'beam',
'L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1b' + extra + 'beam',
'L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1b' + extra + 'beam'
]
#filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1a'+extra+'beam','L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1a'+extra+'beam']
#filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1b'+extra+'beam','L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1b'+extra+'beam']
f = cr.open([BEAMDATA + beams_dir + fname for fname in filenames])
f['CAL_DELAY'] = [
0.0, 7.61718749999e-10, -4.98046875e-09, -8.0078125e-09, -5.0390625e-09,
-4.27734375e-09, 2.92968750002e-10, 7.51953124997e-10, -5.419921875e-09,
-8.408203125e-09, -5.205078125e-09, -5.400390625e-09
] #wrt CS002a, , CC hFFTWResample , ears, new beams with modified rel.ant.pos.
#f['CAL_DELAY']=[0.0,8.00781249999e-10,-4.98046875e-09,-8.046875e-09,-5.05859375e-09,-4.21875e-09,3.12499999998e-10,7.81250000002e-10,-5.4296875e-09,-8.41796875e-09 ,-5.1953125e-09 ,-5.3515625e-09]
#f['CAL_DELAY']=[0.0,8.00781249999e-10,-4.98046875e-09,-8.046875e-09,-5.05859375e-09,-4.21875e-09]
#f['CAL_DELAY']=[3.12499999998e-10,7.81250000002e-10,-5.4296875e-09,-8.41796875e-09 ,-5.1953125e-09 ,-5.3515625e-09]
#f['CAL_DELAY']=[0.0,6.4453125e-10,-5.37109375e-09,-8.4375e-09,-5.2734375e-09,-4.98046875e-09]
RA = 1
#PSR B0329+54 -- Pulsar 0.71452 sec
if RA:
'''Script to test the cross correlation of pulses.
'''
import pycrtools as cr
import numpy as np
import sys; import os
import pdb;# pdb.set_trace()
import Beam_Tools as bt
filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.867Z_CS003_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.887Z_CS004_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1.multi_station.beam']
filenames = [filenames[0],filenames[0]]
#filenames = filenames[0:7:5]
beams = cr.open(filenames)
beams['DM'] = 26.76
block_range=np.arange(240,320)
nblocks = len(block_range)
blocklen = 16384
speclen = 8193
# Do mean of complex or timeseries
mean_fft =1
#pdb.set_trace()
if mean_fft:
cross_correlation = cr.hArray(float,[1,16384]) #beams.empty('TIMESERIES_DATA')[0]
fft_matrix = cr.hArray(complex,[nblocks,speclen]) # ,len(filenames)
'L43784_D20120125T211154.887Z_CS002_R000_tbb.pol0b' + extra[set] +
'beam', 'L43784_D20120125T211154.867Z_CS003_R000_tbb.pol0b' +
extra[set] + 'beam',
'L43784_D20120125T211154.887Z_CS004_R000_tbb.pol0b' + extra[set] +
'beam', 'L43784_D20120125T211154.866Z_CS005_R000_tbb.pol0b' +
extra[set] + 'beam',
'L43784_D20120125T211154.871Z_CS006_R000_tbb.pol0b' + extra[set] +
'beam', 'L43784_D20120125T211154.887Z_CS007_R000_tbb.pol0b' +
extra[set] + 'beam'
],
}
#beams_suffix = '*pol0?'+extra[set]+'beam'
#filenames = glob.glob(BEAMDATA+beams_dir+beams_suffix)
beams = cr.open([BEAMDATA + beams_dir + file for file in filenames[set]])
beams['NCHUNKS'] = 191
#beams['CAL_DELAY']=[-5.4375e-09,-5.96875e-09,0.0,-2.9375e-09,9.375e-11,2.8125e-10] # wrt CS007
#beams['CAL_DELAY']=[5.4375e-09,5.96875e-09,0.0,2.9375e-09,-9.375e-11,-2.8125e-10] # wrt CS007 , negative
#beams['CAL_DELAY']=[0.0,6.875e-10,-5.4375e-09,-8.3125e-09,1.0625e-09,-4.9375e-09] # wrt CS002 ,
#beams['CAL_DELAY']=[0.0,-6.875e-10,5.4375e-09,8.3125e-09,-1.0625e-09,4.9375e-09] # wrt CS002 , negative
#beams['CAL_DELAY']=[0.0,-3.4375e-09,6.25000000003e-10,9.37500000001e-10,9.37500000001e-10,-1.84375e-08] # wrt CS002 , CC values
#beams['CAL_DELAY']=[0.0,-2.5e-09,-3.34375e-09,-3.5625e-09,-3.6875e-09,-1.721875e-08] # wrt CS002 , abs(CC) hFFTWResample
#beams['CAL_DELAY']=[0.0,2.5e-09,3.34375e-09,3.5625e-09,3.6875e-09,1.721875e-08] # wrt CS002 , abs(CC) hFFTWResample , negative
#beams['CAL_DELAY']=[0.0,1.6875e-09,2.375e-09,-5.84375e-09,2.59375e-09,-1.3375e-08] # wrt CS002 , CC hFFTWResample
#beams['CAL_DELAY']=[0.0,1.6859375e-09,2.3625e-09,-5.8421875e-09,2.6046875e-09,-1.3371875e-08] # wrt CS002 , CC hFFTWResample, more acurate
#beams['CAL_DELAY']=[0.0,-1.6875e-09,-2.375e-09,5.84375e-09,-2.59375e-09,1.3375e-08] # wrt CS002 , CC hFFTWResample , negative
#beams['CAL_DELAY']/=2.68 # Magic value :)
#beams['CAL_DELAY']=[0.0,6.4453125e-10,-5.37109375e-09,-8.4375e-09,-5.2734375e-09,-4.98046875e-09 ] #wrt CS002 , CC hFFTWResample , upsampling
#beams['CAL_DELAY']*=-1
#Adding if statement, in case no need to do the "beams" and only the imaging part, ugly fix for now.
#----------------------------------------------------
#Hard coded stuff that will go
filename = '/vol/astro/lofar/frats/tbb/data/L43784_ev1/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5'
polarization = 0
blocklen = 2**14
dm = 26.76
freq_range = [166, 168]
#----------------------------------------------------
#File selection
#Open given TBB file.
fname = filename # args[0]
file = cr.open(fname)
#----------------------------------------------------
#General parameters.
FRATS_ANALYSIS = os.environ["FRATS_ANALYSIS"].rstrip('/') + '/'
Obs_ID = fname.split('/')[-1].split('_')[0]
outdir = FRATS_ANALYSIS + Obs_ID + '/all_sky/'
# Create output directory, if not already present.
if not os.path.isdir(outdir) or not os.path.isdir(outdir + '/beams/'):
os.makedirs(outdir + '/beams/')
if not os.path.isdir(outdir + '/RFI/'):
os.mkdir(outdir + '/RFI/')
'BeamFormer'
) #Using this since it seems the program runs faster when the task is preloaded (if other task is "loaded" then it runs slower).
cr.treset()
if altair:
# f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5')
# filefilter = '/data/FRATS/data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5'
# f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5')
# filefilter = '/data/FRATS/data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5'
# f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.867Z_CS003_R000_tbb.h5')
# filefilter = '/data/FRATS/data/L43784_D20120125T211154.867Z_CS003_R000_tbb.h5'
# f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.887Z_CS004_R000_tbb.h5')
# filefilter = '/data/FRATS/data/L43784_D20120125T211154.887Z_CS004_R000_tbb.h5'
# f1=cr.open('/data/FRATS/data/L43784_D20120125T211154.866Z_CS005_R000_tbb.h5')
# filefilter = '/data/FRATS/data/L43784_D20120125T211154.866Z_CS005_R000_tbb.h5'
f1 = cr.open(
'/data/FRATS/data/L43784_D20120125T211154.887Z_CS007_R000_tbb.h5')
filefilter = '/data/FRATS/data/L43784_D20120125T211154.887Z_CS007_R000_tbb.h5'
output_dir = '/Users/eenriquez/RESEARCH/Pulsars/Results/'
f2 = cr.open(
'/data/FRATS/data/L43784_D20120125T211154.887Z_CS007_R000_tbb.h5')
else:
f1 = cr.open(
'~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5'
)
filefilter = '~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5'
# f1=cr.open('~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5')
# filefilter = '~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.871Z_CS006_R000_tbb.h5'
output_dir = '~/RESEARCH/Pulsars/Results/'
f2 = cr.open(
'~/RESEARCH/Pulsars/Data/L43784_D20120125T211154.887Z_CS002_R000_tbb.h5'
filenames = glob.glob(LOFARDATA + '*.h5')
filter = 'HBA0'
diff = 0
Two_d = 1
if Two_d:
filenames = [filenames[0]]
xpos1 = cr.hArray(float, (len(filenames), 96))
ypos1 = cr.hArray(float, (len(filenames), 96))
zpos1 = cr.hArray(float, (len(filenames), 96))
stations = []
for i, file in enumerate(filenames):
f = cr.open(file)
f['ANTENNA_SET'] = filter
if diff:
antpos = f.getRelativeAntennaPositions(
) - md.getRelativeAntennaPositionsNew(f['STATION_NAME'][0], filter)
else:
antpos = md.getRelativeAntennaPositionsNew(f['STATION_NAME'][0],
filter)
#X-axis positions
xpos = cr.hArray(float, [96])
xpos[...].copy(antpos[..., 0])
xpos1[i] = xpos
#Y-axis positions
Obs_ID ='L74100'
polarization=0
FRATS_ANALYSIS=os.environ["FRATS_ANALYSIS"].rstrip('/')+'/'
##beam_suffix='*pol%i_*HBA?.beam'%(polarization)
#beam_suffix='*pol0_sample*'
beam_suffix='*CS00[2-7]*pol0_sample*'
filenames=glob.glob(FRATS_ANALYSIS+Obs_ID+'_new'+'/beam.results/'+beam_suffix)
#filenames = ['L74100_D20121107T191144.924Z_CS005_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.993Z_CS030_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.993Z_CS030_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.929Z_CS011_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.916Z_CS004_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.929Z_CS011_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.964Z_CS002_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.924Z_CS005_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.964Z_CS002_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.916Z_CS004_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam']
#filenames = ['L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.993Z_CS030_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam','L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam', 'L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam','L74100_D20121107T191144.941Z_CS017_R000_tbb.pol0_sample_calibrated_HBA1.LOFAR_centered.beam','L74100_D20121107T191144.964Z_CS002_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam','L74100_D20121107T191144.917Z_CS007_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam', 'L74100_D20121107T191144.991Z_CS021_R000_tbb.pol0_sample_calibrated_HBA0.LOFAR_centered.beam']
print 'Using files for imaging: ',filenames
#f = cr.open([FRATS_ANALYSIS+Obs_ID+'_new'+'/beam.results/'+filename for filename in filenames])
f = cr.open(filenames)
#PSR B0329+54 -- Pulsar 0.71452 sec
alpha = hms2deg(03,32,59.37); # Right assention
delta = dms2deg(54,34,44.9); # Declination
ctype1 ='RA---SIN'
ctype2 ='DEC--SIN'
f['DM'] = 26.76
#f['CAL_DELAY']=cr.hArray(float, [8], fill=[1.00586e-09,6.32813e-09,-7.97852e-09,9.86328e-09,5.83984e-09,0,-4.26758e-09,3.55469e-09]) # len=24 slice=[0:24])
ST_DELAYS = bt.ccBeams(f,freq_range=[175,195],time_range=[0.346,0.3495],verbose=1)
f['CAL_DELAY']=ST_DELAYS
output = FRATS_ANALYSIS+Obs_ID+'_new'+'/L74100.cal_on.fits'
#! /usr/bin/env python
import pycrtools as cr
import os
import pdb;# pdb.set_trace()
LOFARDATA=os.environ["LOFARDATA"].rstrip('/')+'/'
filenames = ['L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1.multi_station.beam','L43784_D20120125T211154.887Z_CS002_R000_tbb.pol1.multi_station.beam']
f = cr.open([LOFARDATA+fname for fname in filenames])
f['CAL_DELAY']=[0.0,1.00e-12]
f['DM'] = 26.76
output = 'out_pul.sub_st.high_res.fits'
pdb.set_trace()
cr.trun("Imager", data = f, intgrfreq = False, nblocks=30,ntimesteps=10,startblock=100,NAXIS1=33,NAXIS2=33,FREQMIN=1.5e8,FREQMAX=1.69e8,CDELT1=-.125,CDELT2=.125,CTYPE1 ='ALON_SIN',CTYPE2 ='ALAT_SIN',output=output)
"""Run beamformer on Pulsar: /vol/astro/lofar/frats/tbb/data/.
"""
import sys
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pycrtools as cr
from pytmf import *
from pycrtools import tools
from pycrtools.tasks import beamformer
#----------------------------------------------------
fname = sys.argv[1]
f = cr.open(fname)
utc = tools.strdate2jd(f['TIME_HR'][0])
#----------------------------------------------------
outdir = '../beam.results/beams.sub_station/'
substation = 1
inner = 0
detail_name = '.pol0a.new.multi_station'
detail_name2 = '.pol0b.new.multi_station'
#----------------------------------------------------
#Pointing and LOFAR posisition info.
alpha = hms2rad(3, 32, 59.37) # Right assention
delta = dms2rad(54, 34, 44.9) # Declination
phi = deg2rad(52.915122495) #(LOFAR Superterp)
def dyncalc_multibeam(filename=None,
beams=None,
nbeam=0,
save_file=False,
from_file=False,
fraction=None,
tbin=1,
clean=False):
'''
Calculates the dynamic spectrum.
=============== ===== ===================================================================
*beams* None Input array.
*nbeam* 0 Beam to work with, if ()beams has stored multiple ones.
*save_file* False Saves a file in hArray format with additional information besides the array values.
*from_file* False Read cleandynspec from file.
*fraction* None If not None, then a list of the form [x,y] such that extracting the fraction x/y of the data. with x>-1, and y>=x.
*tbin* 1 If >1 integrates over this number of blocks. Or time binning.
*clean* False If True it calculates the cleaned spectrum.
=============== ===== ===================================================================
Example::
import Beam_Tools as bt
dynspec = bt.dyncalc(filename)
or::
import Beam_Tools as bt
dynspec,cleandynspec = bt.dyncalc(beams,tbin=16,clean=True,save_file=True)
The regular and clean dynamic spectra (all blocks) are returned and stored in ``Task.dynspec`` and ``Task.cleandynspec`` respectively.
'''
raise NotImplementedError
if beams == None and filename == None:
raise ValueError(
'Need to provide either the filename or the opened .beam file')
if nbeam != 0 or from_file or save_file:
raise KeyError("Keyword is invalid for now: " + key)
t0 = time.clock()
if beams == None:
beams = cr.open(filename)
elif filename == None:
filename = beams['FILENAMES'][0]
# filename_bin = os.path.join(filename,"data.bin") #Maybe needed in the future if using "from_file" option.
speclen = beams['BLOCKSIZE'] / 2 + 1
block_duration = beams['BLOCKSIZE'] * beams['SAMPLE_INTERVAL'][0]
nblocks = beams['NCHUNKS'] * beams['BEAM_NBLOCKS']
if not fraction:
fraction = [1, 1]
else:
if type(fraction) != type([]) or len(fraction) != 2 or fraction[
0] > fraction[1] or fraction[0] < 0 or fraction[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
if fraction[0] == 0: fraction[0] = 1
nblocks = int(nblocks / fraction[1])
start_time = (fraction[0] - 1) * (block_duration * nblocks) / fraction[1]
end_time = fraction[0] * (block_duration * nblocks) / fraction[1]
beam = beams.empty('FFT_DATA')
tm = cr.hArray(float, beam)
dynspec = cr.hArray(float, [nblocks, beam.shape()[0], speclen])
block_range = range((fraction[0] - 1) * nblocks, (fraction[0]) * nblocks)
pdb.set_trace()
#Reading beams.
for block in block_range:
print ' Calculation at {0:.2%} \r'.format(
float(block) / len(block_range)),
sys.stdout.flush()
beams.getFFTData(beam, block)
tm[...].spectralpower2(beam[...])
dynspec[block] = tm
#Time integration.
if tbin > 1:
dynspec = cr.hArray_toNumpy(dynspec)
dynspec = dynspec.reshape((dynspec.shape[0] / tbin, tbin,
dynspec.shape[1], dynspec.shape[2]))
dynspec = np.sum(dynspec, axis=1)
dynspec = cr.hArray(dynspec)
#Rearranging dimensions.
dynspec = cr.hArray_toNumpy(dynspec)
# dynspec = np.rollaxis(dynspec,0,3)
dynspec = np.swapaxes(dynspec, 0, 1)
dynspec = np.swapaxes(dynspec, 1, 2)
dynspec = cr.hArray(
dynspec.copy()) #The .copy() is quick&dirty way to avoid a bug.
#Cleaning dynamic spectrum.
if clean:
avspec = cr.hArray(float, speclen, fill=0.0)
cleandynspec = cr.hArray(float, dynspec, fill=0.0)
for dim in range(dynspec.shape()[0]):
single_dynspec = cr.hArray(float,
dynspec.shape()[1:], dynspec[dim])
single_dynspec[...].addto(avspec)
cleandynspec[dim] = single_dynspec / avspec
#Create a frequency vector
frequencies = beams['BEAM_FREQUENCIES']
#Create a time vector
times = cr.hArray(
float,
int(round((end_time - start_time) / (block_duration * tbin))),
name="Time",
units=("", "s"))
times.fillrange(start_time, block_duration * tbin)
#Adding parameters
dynspec.par.yvalues = frequencies
dynspec.par.xvalues = times
dynspec.par.tbin = tbin
if clean:
cleandynspec.par.yvalues = frequencies
cleandynspec.par.xvalues = times
cleandynspec.par.tbin = tbin
#Saving file(s).
if save_file:
dynspec.write(os.path.join(filename, "dynspec"),
nblocks=1,
block=0,
clearfile=True)
print 'Saving binary in %s' % os.path.join(filename, "dynspec.pcr")
if clean:
cleandynspec.write(os.path.join(filename, "clean_dynspec"),
nblocks=1,
block=0,
clearfile=True)
print 'Saving binary in %s' % os.path.join(filename,
"clean_dynspec.pcr")
print "Finished - total time used:", time.clock() - t0, "s."
if clean:
return dynspec, cleandynspec
else:
return dynspec
blocksize = int(wide * 1024)
files = args
count = 0
for n in files:
count += 1
data = [array] * count
shiftmax = 0
f = [array] * count
#-------------read in RCUs---------------------------------------
for i in range(0, count, 1):
data[i] = cr.open(files[i], blocksize)
f[i] = cr.TBBData(files[i])
if i == 0:
rcun = data[i]["DIPOLE_NAMES"]
nant = len(rcun)
shifts = zeros((count, nant), dtype=int)
if i != 0:
rcun2 = data[i]["DIPOLE_NAMES"]
rcun = hstack((rcun, rcun2))
del rcun2
#-------------read in antenna positions -------------------------
for i in range(0, count, 1):
beam_suffix)
beam_suffix2 = '*CS00[2-7]*pol%i?.new.beam' % (polarization)
filenames2 = glob.glob(
FRATS_ANALYSIS +
'../testing/PSR_B0329.54/beam.results/beams.sub_station/' +
beam_suffix2)
else:
beam_suffix = '*pol%i.*HBA?.LOFAR_centered.beam' % (polarization)
filenames = glob.glob(FRATS_ANALYSIS + Obs_ID2 + '/beam.results/' +
beam_suffix)
beam_suffix2 = '*pol%i.*HBA?.beam' % (polarization)
filenames2 = glob.glob(FRATS_ANALYSIS + Obs_ID2 + '/beam.results/' +
beam_suffix2)
print 'Using files for calibrating: ', filenames
beams = cr.open(sorted(filenames))
print 'Using files for imaging: ', filenames2
beams2 = cr.open(sorted(filenames2))
#----------------------
#Calibration
#PSR B0329+54 -- Pulsar 0.71452 sec
alpha = hms2deg(03, 32, 59.37)
# Right assention
delta = dms2deg(54, 34, 44.9)
# Declination
ctype1 = 'RA---SIN'
ctype2 = 'DEC--SIN'
def get_noise(event_id):
nstations=0
try:
#for u in np.arange(1):
event = crdb.Event(db=db, id=event_id)
event_time=np.asarray(event["lora_time"])
event_time=event_time[event_time>1.0]
event_time=np.min(event_time)
#if event_time<100.0:
# break
print(event_time)
time = Time(event_time, format='unix', scale='utc',location=loc)
time.delta_ut1_utc = 0.
LST=time.sidereal_time('apparent').hour
print('event time UTC: {0}'.format(event_time))
print('event time LST: {0}'.format(LST))
stations = []
#collect stations with "GOOD" status for event
for f in event.datafiles:
stn=[]
stn.extend(f.stations)
#print stn[0].stationname
#print f.stations.stationname
if stn[0].stationname == "CS001" or stn[0].stationname == "CS002" or stn[0].stationname == "CS003" or stn[0].stationname == "CS004" or stn[0].stationname == "CS005" or stn[0].stationname == "CS006" or stn[0].stationname == "CS007" or stn[0].stationname == "CS011" or stn[0].stationname == "CS013" or stn[0].stationname == "CS017" or stn[0].stationname == "CS021" or stn[0].stationname == "CS026" or stn[0].stationname == "CS028" or stn[0].stationname == "CS030" or stn[0].stationname == "CS031" or stn[0].stationname == "CS032" or stn[0].stationname == "CS101" or stn[0].stationname == "CS103" or stn[0].stationname == "CS301" or stn[0].stationname == "CS302" or stn[0].stationname == "CS401" or stn[0].stationname == "CS501":
if stn[0].status=="GOOD":
stations.extend(f.stations)
nstations=len(stations)
except:
print('no event at this point')
for s in np.arange(nstations):
station_flag=0
station=stations[s]
# The following steps are copied from cr_physics pipeline
# there are a million try/excepts because I ran into lots of specific errors I didn't want to handle
try:
# Open file
f = cr.open(station.datafile.settings.datapath + '/' + station.datafile.filename)
antenna_set= f["ANTENNA_SET"]
# Check if we are dealing with LBA or HBA observations
if "LBA" in f["ANTENNA_SET"]:
print ('LBA event')
else:
print ('HBA event')
continue
except:
print ('no event at antennas')
continue
# Read LORA information
try:
tbb_time = f["TIME"][0]
max_sample_number = max(f["SAMPLE_NUMBER"])
min_sample_number = min(f["SAMPLE_NUMBER"])
(tbb_time_sec, tbb_time_nsec) = lora.nsecFromSec(tbb_time, logfile=os.path.join(lora_directory,lora_logfile))
(block_number_lora, sample_number_lora) = lora.loraTimestampToBlocknumber(tbb_time_sec, tbb_time_nsec, tbb_time, max_sample_number, blocksize=blocksize)
except:
continue
# Check if starting time in sample units (SAMPLE_NUMBER) does not deviate among antennas
try:
sample_number_per_antenna = np.array(f["SAMPLE_NUMBER"])
median_sample_number = np.median(sample_number_per_antenna)
data_length = np.median(np.array(f["DATA_LENGTH"]))
deviating_antennas = np.where( np.abs(sample_number_per_antenna - median_sample_number) > data_length/4)[0]
nof_deviating_antennas = len(deviating_antennas)
print ('Number of deviating antennas: %d' % nof_deviating_antennas)
except:
continue
try:
frequencies = f["FREQUENCY_DATA"]
print ('blocksize: {0}'.format(f["BLOCKSIZE"]))
# Get bandpass filter
nf = f["BLOCKSIZE"] / 2 + 1
ne = int(10. * nf / f["CLOCK_FREQUENCY"])
bandpass_filter.fill(0.)
bandpass_filter[int(nf * 30.0 / 100.)-(ne/2):int(nf * 80.0 / 100.)+(ne/2)] = 1.0
gaussian_weights = cr.hArray(cr.hGaussianWeights(ne, 4.0))
cr.hRunningAverage(bandpass_filter, gaussian_weights)
except:
continue
try:
raw_data = f["TIMESERIES_DATA"].toNumpy()
# Find outliers
tmp = np.max(np.abs(raw_data), axis=1)
outlier_antennas = np.argwhere(np.abs(tmp-np.median(tmp[tmp>0.1])) > 2*np.std(tmp[tmp>0.1])).ravel()
print("Outlier antennas", outlier_antennas)
except:
print 'no raw data'
continue
try:
# Get calibration delays to flag antennas with wrong calibration values
try:
cabledelays = cr.hArray(f["DIPOLE_CALIBRATION_DELAY"])
cabledelays = np.abs(cabledelays.toNumpy())
except:
print 'problem with cable delays'
continue
# Find RFI and bad antennas
findrfi = cr.trun("FindRFI", f=f, nofblocks=10, plotlist=[], apply_hanning_window=True, hanning_fraction=0.2, bandpass_filter=bandpass_filter)
print "Bad antennas", findrfi.bad_antennas
antenna_ids_findrfi = f["SELECTED_DIPOLES"]
nAnt=len(f["SELECTED_DIPOLES"])
bad_antennas_spikes = []
bad_antennas = findrfi.bad_antennas[:]
dipole_names = f["SELECTED_DIPOLES"]
good_antennas = [n for n in dipole_names if n not in bad_antennas]
station["crp_bad_antennas_power"] = findrfi.bad_antennas
station["crp_bad_antennas_spikes"] = bad_antennas_spikes
selected_dipoles = []
for i in range(len(dipole_names) / 2):
if dipole_names[2 * i] in good_antennas and dipole_names[2 * i + 1] in good_antennas and f.nof_consecutive_zeros[2 * i] < 512 and f.nof_consecutive_zeros[2 * i + 1] < 512 and cabledelays[2 * i] < 150.e-9 and cabledelays[2 * i + 1] < 150.e-9:
selected_dipoles.extend([dipole_names[2 * i], dipole_names[2 * i + 1]])
f["SELECTED_DIPOLES"] = selected_dipoles
station["crp_selected_dipoles"] = selected_dipoles
nDipoles=len(selected_dipoles)
except:
print 'issue with RFI'
continue
try:
print block_number_lora
nF= len(frequencies.toNumpy())
all_ffts=np.zeros([nAvg,nDipoles,nF])
all_ffts_cleaned=np.zeros([nAvg,nDipoles,nF])
except:
continue
#______________________________________________________________________
block_number=0
for i in np.arange(nAvg):
try:
# make sure not to include the signal window in the average
if abs(block_number-block_number_lora)<5:
block_number=block_number+10
fft_data = f.empty("FFT_DATA")
#f.getFFTData(fft_data, block_number_lora, True, hanning_fraction=0.2, datacheck=True) # this is what is in the pipeline
f.getFFTData(fft_data, block_number, True, hanning_fraction=0.2, datacheck=True)
# Apply bandpass
fft_data[...].mul(bandpass_filter)
# Normalize spectrum
fft_data /= f["BLOCKSIZE"]
fft_hold=fft_data
# Reject DC component
fft_data[..., 0] = 0.0
# Also reject 1st harmonic (gives a lot of spurious power with Hanning window)
fft_data[..., 1] = 0.0
# Flag dirty channels (from RFI excission)
fft_data[..., cr.hArray(findrfi.dirty_channels)] = 0
# factor of two because reall FFT
all_ffts[i]=2*np.abs(fft_hold.toNumpy())**2
all_ffts_cleaned[i]=2*np.abs(fft_data.toNumpy())**2
badFreq=frequencies.toNumpy()[findrfi.dirty_channels]/1e6
nBadChannelsFilt=len(badFreq[(badFreq>=30.0)*(badFreq<=80.0)])
if nBadChannelsFilt>1:
print 'n bad channels: {0}'.format(nBadChannelsFilt)
continue
block_number=block_number+1
except:
print 'error'
continue
try:
#for y in np.arange(1):
fft_avg=np.average(all_ffts_cleaned,axis=0)
freq=frequencies.toNumpy()
df=(freq[1]-freq[0])/1e6
freq_new=np.arange(30,81,1)
fft_resample=np.zeros([nAnt,nResample])
except:
print 'error in average'
continue
for n in np.arange(nDipoles):
try:
fft_use=fft_avg[n][fft_avg[n]>1e-100]
freq_use=freq[fft_avg[n]>1e-100]
if len(fft_avg[n])>len(fft_use):
station_flag=1
f=interp1d(freq_use/1e6,fft_use)
f_new=f(freq_new)
start_f=np.argmin(np.abs((freq/1e6)-30))
stop_f=np.argmin(np.abs((freq/1e6)-80))
fft_resample[n]=f_new*(1/df)
except:
station_flag=1
print 'issue with interp'
analysisinfo={'event_number': event_id,'station': station.stationname,'UTC_time':event_time,'LST':LST,'frequencies':freq,'FFT_data':fft_avg,'frequencies_50':freq_new,'FFT_data_resampled':fft_resample,'flag': station_flag,'antenna_set':antenna_set,'selected_dipoles': dipole_names,'bad_dipoles':bad_antennas,'nBadChannelsFilt':nBadChannelsFilt}
outputfile=open(station.stationname+'/'+str(int(event_id))+'_noise_OUTER.p','w')
pickle.dump(analysisinfo,outputfile)
outputfile.close()
print '{0} done'.format(station.stationname)
print 'done with event'
#! /usr/bin/env python
'''Images from tbbs.
'''
import pycrtools as cr
from pytmf import *
import os
import pdb;# pdb.set_trace()
LOFARDATA=os.environ["LOFARDATA"].rstrip('/')+'/'
BEAMOUT=os.environ["BEAMOUT"].rstrip('/')+'/'
event_dir ='L43784_ev1/'
filename = 'L43784_D20120125T211154.887Z_CS002_R000_tbb.h5'
f = cr.open(LOFARDATA+event_dir+filename)
RA = 2
#----------------------------------------------------
#RFI excission.
# Find RFI and bad antennas
rfi = cr.trun("FindRFI", f=f,nofblocks=100,verbose=False,startblock=0,save_plots=False)
#print rfi.dirty_channels
#print rfi.good_antennas
if rfi:
f['SELECTED_DIPOLES'] = rfi.good_antennas
#PSR B0329+54 -- Pulsar 0.71452 sec
inner_tiles = options.inner_tiles
polarization = options.polarization
station_centered = options.station_centered
blocklen = options.blocklen
flag_antenna = options.flag_antenna
rfi_find = options.rfi_find
default_input = options.default_input
ref_station = options.ref_station
ref_time = options.ref_time
obs_mode = options.obs_mode
parset_time = options.parset_time
#----------------------------------------------------
#Open given TBB file.
fname = args[0]
file = cr.open(fname)
#----------------------------------------------------
#General parameters logistics.
FRATS_ANALYSIS = os.environ["FRATS_ANALYSIS"].rstrip('/') + '/'
Obs_ID = fname.split('/')[-1].split('_')[0]
Obs_ID2 = fname.split('/')[-1].split('.')[0][:-2] # Minute accuracy.
outdir = FRATS_ANALYSIS + Obs_ID2
# Create output directories, if not already present.
if not os.path.isdir(outdir) or not os.path.isdir(outdir + '/beam.results/'):
os.makedirs(outdir + '/beam.results/')
if not os.path.isdir(outdir + '/RFI/'):
os.mkdir(outdir + '/RFI/')
