def getJonesMatrix(azimuth, zenith, frequencies):
# Produce Jones matrix from antenna model, for position (azimuth, zenith) on the sky
# as a function of 'frequencies' (freq. axis)
#read tables antenna model and initialize values
vt = np.loadtxt(os.environ["LOFARSOFT"] +
"/data/lofar/antenna_response_model/LBA_Vout_theta.txt",
skiprows=1)
vp = np.loadtxt(os.environ["LOFARSOFT"] +
"/data/lofar/antenna_response_model/LBA_Vout_phi.txt",
skiprows=1)
cvt = cr.hArray(vt[:, 3] + 1j * vt[:, 4])
cvp = cr.hArray(vp[:, 3] + 1j * vp[:, 4])
fstart = 10.0 * 1.e6 # Default numbers used for computing Jones matrix from antenna model
fstep = 1.0 * 1.e6
fn = 101
ttstart = 0.0
ttstep = 5.0
ttn = 19
pstart = 0.0
pstep = 10.0
pn = 37
jones_matrix = cr.hArray(complex, dimensions=(len(frequencies), 2, 2))
for k, f in enumerate(frequencies):
if (f > 1e7 and f < 1e8):
cr.hGetJonesMatrix(jones_matrix[k], f,
180 - (azimuth / np.pi * 180),
90 - (zenith / np.pi * 180), cvt, cvp, fstart,
fstep, fn, ttstart, ttstep, ttn, pstart, pstep,
pn)
jm = jones_matrix.toNumpy()
return jm
def super_plot(zoom_dynspec, pulse=0, zoom=1, info=None):
'''Do some awesome ploting.
'''
#Create a frequency vector
frequencies = zoom_dynspec.par.yvalues
#Create a time vector
times = zoom_dynspec.par.xvalues
pdb.set_trace()
cr.plt.clf()
cr.plt.ion()
zoom = 1
if zoom:
cr.plt.rcParams['font.size'] = 20
cr.plt.rcParams['axes.titlesize'] = 30
cr.plt.subplot(2, 2, 1)
cr.plt.title("Pulse Profile from : " + zoom_dynspec.par.station)
cr.plt.xlabel("+/- Time [s]")
flat_dyn = bt.flatdyn(zoom_dynspec, axis='x')
cr.plt.plot(flat_dyn.par.xvalues, flat_dyn)
cr.plt.autoscale(axis='x', tight=True)
cr.plt.subplot(2, 2, 4)
cr.plt.xlabel("Frequency [MHz]")
flat_dyn, flat_dyn2 = bt.flatdyn(
zoom_dynspec,
axis='y',
pulse_range=info.par.pulse_range,
background_range=[0, 50 / info.par.tbin])
cr.plt.plot(flat_dyn.par.xvalues, flat_dyn)
cr.plt.plot(flat_dyn.par.xvalues, flat_dyn2, 'r')
cr.plt.autoscale(axis='x', tight=True)
cr.plt.subplot(2, 2, 3)
#Time integration.
tbin = 1
if tbin > 1:
zoom_dynspec = cr.hArray_toNumpy(zoom_dynspec)
zoom_dynspec = zoom_dynspec.reshape(
(zoom_dynspec.shape[0] / tbin, tbin, zoom_dynspec.shape[1]))
zoom_dynspec = np.sum(zoom_dynspec, axis=1)
zoom_dynspec = cr.hArray(zoom_dynspec)
zoom_dynspec = zoom_dynspec.Transpose()
tbin = 4
if tbin > 1:
zoom_dynspec = cr.hArray_toNumpy(zoom_dynspec)
zoom_dynspec = zoom_dynspec[:-3]
zoom_dynspec = zoom_dynspec.reshape(
(zoom_dynspec.shape[0] / tbin, tbin, zoom_dynspec.shape[1]))
zoom_dynspec = np.sum(zoom_dynspec, axis=1)
zoom_dynspec = cr.hArray(zoom_dynspec)
zoom_dynspec = zoom_dynspec.Transpose()
cr.plt.imshow(np.log10(cr.hArray_toNumpy(zoom_dynspec)),
aspect='auto',
origin='lower',
cmap=cr.plt.cm.hot,
extent=(times[0], times[-1], frequencies[0],
frequencies[-1]))
cr.plt.xlabel("+/- Time [s]")
cr.plt.ylabel("Frequency [MHz]")
def GetDistance(core, direction, positions):
"""
Calculating the distance of a station to the shower core in the shower plane.
"""
positions = cr.hArray(positions)
core = cr.hArray(core)
distances = cr.hArray(copy=positions)
finaldistance = cr.hArray(copy=positions)
theta = cr.radians(
direction[1]) # spherical coordinates: 0 = horizon, 90 = vertical
phi = cr.radians(450 -
direction[0]) # recalculate to 0 = east counter-clockwise
distances = positions - core
axis = cr.hArray([
cr.cos(theta) * cr.cos(phi),
cr.cos(theta) * cr.sin(phi),
cr.sin(theta)
]) # shower axis
finaldistance[...].crossproduct(distances[...], axis)
dist = cr.hVectorLength(finaldistance[...])
return dist
def flatDynspec(dynspec, axis='x', verbose=False, down_sample=0):
'''Integrates the dynspec over time or freq.
================== ===== ===================================================================
*dynspec* Dynamic spectrum produced either by addBeams, cutDynspec or rawdyncalc.
*axis* 'x' Axis to integrate over, 'x' for freq integration, 'y' for time integration.
*verbose* False To plot or print extra info.
*down_sample* 0 Downsample factor, if 0, then not downsampling.
================== ===== ===================================================================
Example::
import Beam_Tools as bt
flat_cleandyn=bt.flatDynspec(zoom_dynspec,axis='x', verbose=True, down_sample=20)
'''
#-----------
#Things to check log:
#1)
#-----------
ax = cr.hArray()
if axis == 'y':
ax.par.xvalues = dynspec.par.yvalues
x_label = 'Frequency [Hz]'
elif axis == 'x':
ax.par.xvalues = dynspec.par.xvalues
dynspec = dynspec.Transpose()
x_label = 'Time [s]'
ax.par.flattype = axis
flat_dynspec = cr.hArray(dynspec[...].sum())
flat_dynspec.par = ax.par
if down_sample:
#Downsampling
flat_sampled = cr.hDownsample(flat_dynspec.vec(), down_sample)
flat_sampled = cr.hArray(float, len(flat_dynspec.par.xvalues),
flat_sampled)
flat_sampled.par = ax.par
flat_dynspec = flat_sampled
if verbose:
cr.plt.ion()
cr.plt.clf()
cr.plt.plot(flat_dynspec.par.xvalues, flat_dynspec.vec())
cr.plt.xlabel(x_label)
return flat_dynspec
def run(self):
if getFFTData:
self.offset = cr.hArray(self.block + shifts[self.i])
data[self.i].getFFTData(fftdata[self.i * nant:(self.i + 1) * nant],
self.offset)
else:
self.offset = cr.hArray(self.block + shifts[self.i])
self.offset = cr.hArray((self.block) * blocksize + shifts[self.i] +
startblock)
# data[self.i]["BLOCKSIZE"] = blocksize
# print "BLA", fftdata.shape(), self.i*nant, (self.i+1)*nant, self.offset
# data[self.i].getTimeseriesData(fftdata[self.i*nant:(self.i+1)*nant], self.offset)
cr.hReadTimeseriesData(fftdata[self.i * nant:(self.i + 1) * nant],
self.offset, blocksize, f[self.i])
def fitBaseline(flat_dynspec, order=10):
'''This program fits the baseline.
'''
flat_numpy_dynspec = cr.hArray(float, flat_dynspec, flat_dynspec)
flat_numpy_dynspec = flat_numpy_dynspec.toNumpy()
x_numpy_values = flat_dynspec.par.xvalues.toNumpy()
numpy_coeffs = np.polyfit(x_numpy_values, flat_numpy_dynspec, order)
numpy_fit = np.polyval(numpy_coeffs, x_numpy_values)
flat_fit = cr.hArray(numpy_fit)
flat_fit.sqrt()
flat_fit = 1 / flat_fit
return flat_fit
def getRADEC_2_AZEL(alpha, delta, utc, angl_offset=0.0):
''' Quick funciton to change coordinates from RADEC to AZEL.
'''
phi = deg2rad(52.915122495) #(LOFAR Superterp)
L = deg2rad(6.869837540)
angl_offset = deg2rad(angl_offset)
alpha += angl_offset
#delta = delta+angl_offset
#----------------------------------------------------
# Make input and output arrays for conversion
equatorial = cr.hArray([alpha, delta])
horizontal = equatorial.new()
#----------------------------------------------------
# Convert all coordinates in the input array
# assumes difference between UTC and UT is 0 (hence ut1_utc=0.)
cr.hEquatorial2Horizontal(horizontal, equatorial, utc, 0., L, phi)
if horizontal[0] < 0:
horizontal[
0] += 2. * np.pi # Need to check the definitions used for positive azimut angles.
azel = [horizontal[0], horizontal[1]]
pointings = [{'az': azel[0], 'el': azel[1]}]
return pointings
def doStats(data, verbose=True, nof_St=0):
''' Do simple SNR calculation.
============== ===== ===================================================================
*data* One dimentional vector with the frequency integrated spectrum.
*verbose* True Print the SNR results.
*nof_St* 0 Use str(beams['NOF_STATION_BEAM']), to show the number of stations.
============== ===== ===================================================================
Example::
import Beam_Tools as bt
stats = bt.doStats(flat_cleandyn,verbose=True,nof_St=str(beams['NOF_BEAM_DATASETS']))
'''
stats = cr.hArray(float, 6, fill=0)
data = cr.hArray(float, len(data), data)
lenght = len(data)
mask1 = int(lenght / 5)
mask2 = [int(lenght / 100) if lenght / 100 > 3 else 3][0]
data.sort()
stats[0] = data[:-1 * mask1].vec().mean() # Mean without the peak.
data -= stats[0]
stats[1] = data[-1 * mask2:].vec().mean() # Max value
stats[2] = data[:-1 * mask1].vec().stddev() # Stddev
stats[3] = stats[1] / stats[2] # SNR
stats[4] = data.vec().min()
stats[5] = data.vec().max()
if verbose:
print '----------------------------------'
print 'SNR calculations.'
print '----------------------------------'
print 'Original Mean ', stats[0]
print 'Min - OM', stats[4]
print 'Max - OM', stats[1]
print 'Stddev ', stats[2]
print '....................'
print 'For %i stations' % (nof_St)
print 'SNR (max_range/stddev)', stats[3]
print 'SNR (max/stddev)', stats[5] / stats[2]
print '----------------------------------'
return stats
def dostats(data):
stats = cr.hArray(float, 4, fill=0)
data = cr.hArray(float, len(data), data)
lenght = len(data)
mask1 = lenght / 5
mask2 = lenght / 20
data.sort()
stats[0] = data[:-1 * mask1].vec().mean() # Mean without the peak.
data -= stats[0]
stats[1] = data[-1 * mask2:].vec().mean() # Max value
stats[2] = data[:-1 * mask1].vec().stddev() # Stddev
stats[3] = stats[1] / stats[2] # SNR
return stats
def azel2beamweights(azel, station, freq, antennaset, lofarcentered=False):
#antennaset = HBA0, HBA1 or HBA_DUAL
if lofarcentered:
antpos = md.getLofarCenterRelativeAntennaPosition(
station, antennaset, True)
else:
antpos = md.getStationCenterRelativeAntennaPosition(
station, antennaset, True)
delays = cr.hArray(float, [96])
pointings = cr.hArray([cr.convert(coords, "CARTESIAN")
for coords in azel][0])
cr.hGeometricDelays(delays, antpos, pointings, True)
phases = cr.hArray(float, [96])
phases.delaytophase(cr.hArray([freq]), delays)
weights = cr.hArray(complex, 96)
weights.phasetocomplex(phases)
return weights
def cutbeam(beam, axis='xy', freq_range=[0, 0], startblock=0, nblocks=1):
'''Cuts a beam around the dedispersed peak.
'''
raise NotImplementedError
if not np.any(freq_range):
freq_range = [100, 200]
if not nblocks:
if axis == 'xy':
nblocks = 640
else:
nblocks = beam.shape()[0]
speclen = 8193
freq_range = cr.hArray(float, [2], fill=freq_range,
units=('M', 'Hz')).setUnit("", "Hz")
frequencies = cr.hArray(
float, speclen, list(np.arange(1e+08, 2e+08 + 12207.03125,
12207.03125)))
frequency_slice = cr.hArray(int, 2)
cr.hFindSequenceBetweenOrEqual(frequency_slice, frequencies, freq_range[0],
freq_range[1], 0, 0)
if axis == 'xy':
# sliced_beam = cr.hArray(complex,[nblocks,beam.shape()[1]],beam)
sliced_beam = cr.hArray(float, [nblocks, beam.shape()[1]],
beam[startblock:nblocks + startblock])
else:
sliced_beam = beam
T_beam = sliced_beam.Transpose()
zoom_beam = cr.hArray(complex,
[frequency_slice[1] - frequency_slice[0], nblocks],
T_beam[frequency_slice[0]:frequency_slice[1]])
zoom_beam = zoom_beam.Transpose()
times = cr.hArray(
float, nblocks,
list(
np.arange(startblock * 8192 * 2 * 5e-9,
(startblock + nblocks) * 8192 * 2 * 5e-9,
8192 * 2 * 5e-9)))
frequencies = cr.hArray(
float, frequency_slice[1] - frequency_slice[0],
frequencies[frequency_slice[0]:frequency_slice[1]].vec())
zoom_beam.par.xvalues = times
zoom_beam.par.yvalues = frequencies
return zoom_beam
def getRef_time(filelist, fullparsetname=''):
''' Gets the reference time for a list of tbb files (station which start observing last).
'''
times = cr.hArray(float, len(filelist), 0.0)
seconds = cr.hArray(float, len(filelist), 0.0)
for i, file in enumerate(filelist):
tbb = cr.open(file)
if fullparsetname:
f_clock_offset = float(
md.getClockCorrectionParset(fullparsetname,
tbb['STATION_NAME'][0],
antennaset=tbb['ANTENNA_SET']))
else:
f_clock_offset = float(
md.getClockCorrection(tbb['STATION_NAME'][0],
antennaset=tbb['ANTENNA_SET']))
time_tbb = cr.hArray(float,
len(tbb['SAMPLE_NUMBER']),
fill=tbb['TIME'])
time_tbb -= min(tbb['TIME'])
sample_tbb = cr.hArray(float,
len(tbb['SAMPLE_NUMBER']),
fill=tbb['SAMPLE_NUMBER'])
sample_tbb *= tbb['SAMPLE_INTERVAL'][0]
sample_tbb += time_tbb
times[i] = max(sample_tbb) + f_clock_offset
seconds[i] = min(tbb['TIME'])
tbb.close()
if verbose:
print 'Reference time: ', max(times), 'from file ', filelist[
cr.hFindBetweenOrEqual(times, max(times), max(times))[0]]
return [min(seconds), max(times)]
def get_sample_offset(
fullparsetname,
file,
antenna_set,
max_sample_number,
sample_calibrated=False,
ref_station='',
ref_time=None,
):
#----------------------------------------------------
#Calibrate station times.
f_clock_offset = float(
md.getClockCorrectionParset(fullparsetname,
file['STATION_NAME'][0],
antennaset=antenna_set))
blocklen = file['BLOCKSIZE']
if not sample_calibrated:
#First step: Rounding the sample_number used to the nearest whole block since second start, including CLOCK_OFFSET.
sample_offset = cr.hArray(int, 1, max_sample_number)
cr.hModulus(sample_offset, blocklen)
sample_offset = cr.asval(blocklen - sample_offset +
int(f_clock_offset /
file['SAMPLE_INTERVAL'][0]))
else:
if not ref_time:
if not ref_station:
f0 = file
else:
f0 = cr.open(ref_station)
print 'WARNING, using a ref_station assumes the dumps were within the same second, if not then give a ref_time.'
f0_clock_offset = float(
md.getClockCorrectionParset(fullparsetname,
f0['STATION_NAME'][0],
antennaset='HBA0'))
t0 = max(f0['SAMPLE_NUMBER']
) * f0['SAMPLE_INTERVAL'][0] + f0_clock_offset
else:
t0 = ref_time[1]
t = max_sample_number + f_clock_offset
sample_offset = [
int((t0 - t) / file['SAMPLE_INTERVAL'][0]),
(t0 - t) / file['SAMPLE_INTERVAL'][0] - int(
(t0 - t) / file['SAMPLE_INTERVAL'][0])
]
return sample_offset
def getPulseInfo(pulse=0, tbin=1):
'''Used to get the freq and time ranges to cut the pulse.
'''
dt_pulse = .005
if pulse == 1:
freq_range = [132, 146] # 130#145
time_range = [0.57, 0.6201]
pulse_range = [.5915, .5915 + dt_pulse]
#pulse_range = [220/tbin,270/tbin]
#frequency_slice = [2458,3687]# [2622,3769]
# freq_size=1147+1 #1228
# frequency_slice = [2622,3770] #[2458,3686]
# nblocks=560
# startblock=7000
elif pulse == 3: #pulse 3
freq_range = [149, 169] #[155,160]#
time_range = [0.0, 0.0501]
pulse_range = [.0205, .0205 + dt_pulse]
#pulse_range = [250/tbin,300/tbin]
#frequency_slice = [4015,5653] [4506,4916]
# nblocks=640
# startblock=0
elif pulse == 2: #pulse 2
freq_range = [140, 155]
time_range = [0.28, 0.3301]
pulse_range = [.308, .308 + dt_pulse]
elif pulse == 4: #pulse 4
freq_range = [126, 136]
time_range = [.86, .9101]
pulse_range = [.881, .881 + dt_pulse]
elif pulse == 5: #pulse 5
freq_range = [106, 117]
time_range = [0.715, 0.7651]
pulse_range = [.739, .739 + dt_pulse]
elif pulse == 6: #pulse 6
freq_range = [178, 190]
time_range = [0.415, 0.4651]
pulse_range = [.435, .435 + dt_pulse]
elif pulse == 7: #pulse 7
freq_range = [126, 136]
info_pulse = cr.hArray()
info_pulse.par.freq_range = freq_range
info_pulse.par.time_range = time_range
info_pulse.par.pulse_range = pulse_range
return info_pulse
def findRFI(dynspec, return_more=False, single_plot=False):
'''
Find RFI in dynspec by sum(square(v)) / square(sum(v))
with the sum over the blocks for each frequency.
============== ===== ===================================================================
*dynspec*
*return_more*
*plot* List with beam index [n],
============== ===== ===================================================================
Example::
import Beam_Tools as bt
RFI_channels = bt.findRFI(dynspec)
'''
if len(dynspec.shape()) == 2:
dynspec.reshape([1, dynspec.shape()[0], dynspec.shape()[1]])
dynspec_sqr = cr.hArray(float, dynspec.shape(), dynspec)
dynspec_sqr.square()
sum_sqr = cr.hArray(float, [dynspec.shape()[0], dynspec.shape()[1]])
sqr_sum = cr.hArray(float, [dynspec.shape()[0], dynspec.shape()[1]])
#adding blocks
for beam in range(dynspec.shape()[0]):
sqr_sum[beam, ...] = dynspec[beam, ...].sum()
sum_sqr[beam, ...] = dynspec_sqr[beam, ...].sum()
#Squaring the sums
sqr_sum.square()
#Define ratio
dynspec_ratio = sum_sqr / sqr_sum
if dynspec.shape()[1] > 10:
channel_cut = dynspec.shape()[1] / 10
else:
channel_cut = 1
RFI_channels = []
#Define thresholds
for beam in range(dynspec.shape()[0]):
scrash_vec = cr.hArray(float,
dynspec_ratio.shape()[1],
sorted(dynspec_ratio[beam].vec()))
sigma = scrash_vec[channel_cut:-1 * channel_cut].stddev()
RFI_threshold1 = scrash_vec.median() + 4 * sigma
RFI_threshold2 = scrash_vec.median() - 4 * sigma
#Find RFI
RFI_1 = dynspec_ratio[beam].vec().Find('>', RFI_threshold1[0])
RFI_2 = dynspec_ratio[beam].vec().Find('<', RFI_threshold2[0])
RFI_channels.append(sorted(list(set(RFI_1) | set(RFI_2))))
if beam == single_plot[0]:
cr.plt.ion()
cr.plt.plot(dynspec_ratio[beam].vec())
cr.plt.plot([0, dynspec.shape()[1]],
[RFI_threshold1[0], RFI_threshold1[0]])
cr.plt.plot([0, dynspec.shape()[1]],
[RFI_threshold2[0], RFI_threshold2[0]])
if return_more:
return RFI_channels, dynspec_ratio
else:
return RFI_channels
def showPulseProfiles(beams,
freq_range=[130, 160],
time_range=[0, 0.1],
doplot=False,
tbin=16):
''' Show pulse profile for each beam.
============== ===== ===================================================================
*beams* Input beams, opened by the beam.py interphase.
*freq_range* None Frequency range in MHz
*time_range* None Time range in seconds
*tbin* 16 If >1 integrates over this number of blocks. Or time binning.
*doplot* False If True then it plots the flat spectra of each beam.
============== ===== ===================================================================
Example::
import Beam_Tools as bt
dynspecs,flat_dynspecs = bt.showPulseProfiles(beams,freq_range=[175,195],time_range=[0.1,0.2])
'''
Obs_ID = beams['TBB_FILENAME'].split('/')[-1].split('_')[0]
#Dynspec calc.
print 'Calculating single beam dynamic spectra.'
dynspec, cleandynspec = calcIndividualStationDynspecs(beams,
tbin=tbin,
clean=True)
dynspec = cleandynspec
SNR = cr.hArray(float, dynspec.shape()[0])
if doplot:
cr.plt.ion()
#Cut and Integrate frequencies.
all_flat_dynspec = cr.hArray(
float, [dynspec.shape()[0], dynspec.shape()[2]])
for dyn in range(dynspec.shape()[0]):
zoom_dynspec = cutDynspec(dynspec[dyn],
freq_range=freq_range,
time_range=time_range)
flat_dynspec = flatDynspec(zoom_dynspec, axis='x')
stats = doStats(flat_dynspec, verbose=False)
SNR[dyn] = stats[5] / stats[2]
print ' ' * 2
print 'The SNR in ' + beams['STATION_NAME'][dyn] + '_' + beams[
'ANTENNA_SET'][dyn] + ' is:', SNR[dyn]
#ploting
if doplot:
times = zoom_dynspec.par.xvalues
cr.plt.plot(times,
flat_dynspec / flat_dynspec.vec().mean() + dyn * 0.1,
label=beams['STATION_NAME'][dyn] + '_' +
beams['ANTENNA_SET'][dyn] + ', SNR = ' + str(SNR[dyn]))
all_flat_dynspec[dyn, ...] = flat_dynspec.vec()
if doplot:
cr.plt.rcParams['font.size'] = 20
cr.plt.rcParams['axes.titlesize'] = 30
cr.plt.xlabel('Relative Time [s]')
cr.plt.ylabel('Power [Relative units]')
cr.plt.title(Obs_ID + ': Pulse profile for individual beams.')
cr.plt.legend()
return dynspec, all_flat_dynspec
def calcIndividualStationDynspecs(beams,
dm=0,
save_file=False,
fraction=None,
tbin=1,
clean=False,
verbose=1,
time_range=None):
'''
Calculates the dynamic spectrum of individual beams.
============ ===== ===================================================================
*beams* Input array.
*dm* 0 Dedispersion Measure
*save_file* False Saves a file in hArray format with additional information besides the array values.
*fraction* None If not None, then a list of the form [x,y] such that extracting the fraction x/y of the data. with x>-1, and y>=x.
*tbin* 1 If >1 integrates over this number of blocks. Or time binning.
*clean* False If True it calculates the cleaned spectrum.
*verbose* 1 If want to print status and performance.
*time_range* None List with the begin and end of time selection. [t1,t2] in s
============ ===== ===================================================================
Example::
import Beam_Tools as bt
dynspecs = bt.calcIndividualStationDynspecs(beams)
or::
import Beam_Tools as bt
dynspecs,cleandynspecs = bt.calcIndividualStationDynspecs(beams,tbin=16,clean=True,save_file=True,time_range=[0,.1])
'''
#-----------
#Log of things to check:
#1)tbin divisiion
#-----------
t0 = time.clock()
# filename_bin = os.path.join(filename,"data.bin") #Maybe needed in the future if using "from_file" option.
speclen = beams['BEAM_SPECLEN']
block_duration = beams['BLOCKSIZE'] * beams['SAMPLE_INTERVAL'][0]
nblocks = beams['NCHUNKS'] * beams['BEAM_NBLOCKS']
if not beams['DM'] and dm:
beams['DM'] = dm
if time_range:
#Time selection indexing.
if type(time_range) != type([]) or len(time_range) != 2 or time_range[
0] > time_range[1] or time_range[0] < 0 or time_range[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
times = beams.getTimes()
block_range = cr.hArray(int, 2)
cr.hFindSequenceBetweenOrEqual(block_range, times, time_range[0],
time_range[1], 0, 0)
nblocks = block_range[1] - block_range[0]
fraction = False
block_range = range(block_range[0], block_range[1])
if not fraction:
fraction = [1, 1]
if not time_range:
block_range = range(0, nblocks)
else:
if type(fraction) != type([]) or len(fraction) != 2 or fraction[
0] > fraction[1] or fraction[0] < 0 or fraction[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
if fraction[0] == 0: fraction[0] = 1
nblocks = int(nblocks / fraction[1])
block_range = range((fraction[0] - 1) * nblocks,
(fraction[0]) * nblocks)
beam = beams.empty('FFT_DATA')
dynspec = cr.hArray(float, [beam.shape()[0], nblocks, beam.shape()[1]])
#Dynamic spectrum calculation.
for i, block in enumerate(block_range):
if verbose:
print 'Dynamic spectrum calculation at {0:.2%} \r'.format(
float(i) / nblocks),
sys.stdout.flush()
beams.getFFTData(beam, block)
for b in range(beam.shape()[0]):
dynspec[b, i].spectralpower2(beam[b])
#Time integration.
if tbin > 1:
dynspec = cr.hArray_toNumpy(dynspec)
dynspec = dynspec.reshape((dynspec.shape[0], dynspec.shape[1] / tbin,
tbin, dynspec.shape[2]))
dynspec = np.sum(dynspec, axis=2)
dynspec = cr.hArray(dynspec)
#Cleaning dynamic spectrum.
if clean:
cleandynspec = dynspec * 0
for b in range(beam.shape()[0]):
avspec = cr.hArray(float, speclen, fill=0.0)
dynspec[b, ...].addto(avspec)
cleandynspec[b] = (dynspec[b] * dynspec.shape()[1] / avspec)[0]
#Transposing arrays.
dynspec_T = dynspec.Transpose() * 0
for b in range(beam.shape()[0]):
dynspec_T[b] = dynspec[b].Transpose()
dynspec = dynspec_T
if clean:
cleandynspec_T = cleandynspec.Transpose() * 0
for b in range(beam.shape()[0]):
cleandynspec_T[b] = cleandynspec[b].Transpose()
cleandynspec = cleandynspec_T
#Create a frequency vector
frequencies = beams['BEAM_FREQUENCIES']
#Create a time vector
start_time = block_range[0] * block_duration
end_time = block_range[-1] * block_duration
times = cr.hArray(
float,
int(round((end_time - start_time) / (block_duration * tbin))),
name="Time",
units=("", "s"))
times.fillrange(start_time, block_duration * tbin)
#Adding parameters
dynspec.par.yvalues = frequencies
dynspec.par.xvalues = times
dynspec.par.tbin = tbin
if clean:
cleandynspec.par.yvalues = frequencies
cleandynspec.par.xvalues = times
cleandynspec.par.tbin = tbin
#Saving file(s).
if save_file:
dynspec.write(os.path.join(filename, "dynspec"),
nblocks=1,
block=0,
clearfile=True)
print 'Saving binary in %s' % os.path.join(filename, "dynspec.pcr")
if clean:
cleandynspec.write(os.path.join(filename, "clean_dynspec"),
nblocks=1,
block=0,
clearfile=True)
print 'Saving binary in %s' % os.path.join(filename,
"clean_dynspec.pcr")
if verbose:
print "Finished - dyncalc time used:", time.clock() - t0, "s."
if clean:
return dynspec, cleandynspec
else:
return dynspec
def rawdyncalc(TAB,
fraction=None,
tbin=1,
clean=False,
axis_val=False,
verbose=1):
'''
Calculates the dynamic spectrum of a raw beam (no TBB info).
=============== ===== ===================================================================
*TAB* Input complex array.
*fraction* None If not None, then a list of the form [x,y] such that extracting the fraction x/y of the data. with x>-1, and y>=x.
*tbin* 1 If >1 integrates over this number of blocks. Or time binning.
*clean* False If True it calculates the cleaned spectrum.
*verbose* 1 If want to print status and performance.
=============== ===== ===================================================================
Example::
import Beam_Tools as bt
dynspec = bt.rawdyncalc(filename)
or::
import Beam_Tools as bt
dynspec,cleandynspec = bt.rawdyncalc(TAB,tbin=16,clean=True)
'''
t0 = time.clock()
speclen = TAB.shape()[1]
block_duration = (TAB.shape()[1] - 1) * 2 * 5e-09
nblocks = TAB.shape()[0]
#Create a frequency vector
if axis_val:
frequencies = TAB.par.yvalues
else:
frequencies = cr.hArray(
float, speclen,
list(np.arange(1e+08, 2e+08 + 12207.03125, 12207.03125)))
if not fraction:
fraction = [1, 1]
else:
if type(fraction) != type([]) or len(fraction) != 2 or fraction[
0] > fraction[1] or fraction[0] < 0 or fraction[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
if fraction[0] == 0: fraction[0] = 1
nblocks = int(nblocks / fraction[1])
start_time = (fraction[0] - 1) * (block_duration * nblocks) / fraction[1]
end_time = fraction[0] * (block_duration * nblocks) / fraction[1]
dynspec = cr.hArray(float, [nblocks, speclen])
block_range = range((fraction[0] - 1) * nblocks, (fraction[0]) * nblocks)
beam = cr.hArray(complex, speclen)
#Reading TAB.
for block in block_range:
if verbose:
print 'Dynspec calculation at {0:.2%} \r'.format(
float(block) / nblocks),
sys.stdout.flush()
beam = TAB[block]
dynspec[block].spectralpower2(beam)
#Time integration.
if tbin > 1:
dynspec = cr.hArray_toNumpy(dynspec)
dynspec = dynspec.reshape(
(dynspec.shape[0] / tbin, tbin, dynspec.shape[1]))
dynspec = np.sum(dynspec, axis=1)
dynspec = cr.hArray(dynspec)
#Cleaning dynamic spectrum.
if clean:
avspec = cr.hArray(float, speclen, fill=0.0)
dynspec[...].addto(avspec)
cleandynspec = dynspec / avspec
#Transposing arrays.
dynspec = dynspec.Transpose()
if clean:
cleandynspec = cleandynspec.Transpose()
#Create a time vector
if axis_val:
times = TAB.par.xvalues
else:
times = cr.hArray(
float,
int(round((end_time - start_time) / (block_duration * tbin))),
name="Time",
units=("", "s"))
times.fillrange(start_time, block_duration * tbin)
#Adding parameters
dynspec.par.yvalues = frequencies
dynspec.par.xvalues = times
dynspec.par.tbin = tbin
if clean:
cleandynspec.par.yvalues = frequencies
cleandynspec.par.xvalues = times
cleandynspec.par.tbin = tbin
if verbose:
print "Finished - dyncalc time used:", time.clock() - t0, "s."
if clean:
return dynspec, cleandynspec
else:
return dynspec
def write_dynspec(self):
"""
Saving Beamformed FFT data into a .beam file
"""
# Writing header
#f = h5py.File(self.infile[0], 'r')
f = self.bffile
ndipoles = f.attrs["NOF_DIPOLE_DATASETS"]
self.__bf_dictionary()
# Writing data
print("Writing FFT", self.station, "with shape", self.fftdata.shape)
hfftdata = cr.hArray(self.fftdata, name="Beamed FFT")
hfftdata.setHeader(
# BeamFormer keywords
BeamFormer=self.bf_dict,
# Existing keywords
OBSERVATION_START_UTC=str(f.attrs["OBSERVATION_START_UTC"]),
OBSERVATION_ID=str(f.attrs["OBSERVATION_ID"]),
CLOCK_FREQUENCY_UNIT=str(f.attrs["CLOCK_FREQUENCY_UNIT"]),
NOTES=str(f.attrs["NOTES"]),
OBSERVATION_FREQUENCY_CENTER=f.
attrs["OBSERVATION_FREQUENCY_CENTER"],
PROJECT_PI=str(f.attrs["PROJECT_PI"]),
OBSERVATION_END_UTC=str(f.attrs["OBSERVATION_END_UTC"]),
PROJECT_CO_I=str(f.attrs["PROJECT_CO_I"]),
TELESCOPE=str(f.attrs["TELESCOPE"]),
ANTENNA_SET=str(f.attrs["ANTENNA_SET"]),
OBSERVATION_START_MJD=f.attrs["OBSERVATION_START_MJD"],
PROJECT_CONTACT=str(f.attrs["PROJECT_CONTACT"]),
FILTER_SELECTION=str(f.attrs["FILTER_SELECTION"]),
FILETYPE=str(f.attrs["FILETYPE"]),
OBSERVATION_FREQUENCY_MAX=f.attrs["OBSERVATION_FREQUENCY_MAX"],
CLOCK_FREQUENCY=f.attrs["CLOCK_FREQUENCY"],
OBSERVATION_END_MJD=f.attrs["OBSERVATION_END_MJD"],
OBSERVATION_NOF_STATIONS=f.attrs["OBSERVATION_NOF_STATIONS"],
OBSERVATION_FREQUENCY_UNIT=str(
f.attrs["OBSERVATION_FREQUENCY_UNIT"]),
SYSTEM_VERSION=str(f.attrs["SYSTEM_VERSION"]),
OBSERVATION_FREQUENCY_MIN=f.attrs["OBSERVATION_FREQUENCY_MIN"],
PROJECT_ID=str(f.attrs["PROJECT_ID"]),
PROJECT_TITLE=str(f.attrs["PROJECT_TITLE"]),
FILEDATE=str(f.attrs["FILEDATE"]),
FILENAME=self.dynspecfile.split('/')[-1],
TARGET=f.attrs["TARGETS"],
# Keywords derived in the beamforming part
STATION_NAME=[str(f.attrs["STATION_NAME"])] * ndipoles,
DIPOLE_NAMES=f.attrs["DIPOLE_NAMES"],
NOF_SELECTED_DATASETS=f.attrs["NOF_SELECTED_DATASETS"],
NOF_DIPOLE_DATASETS=f.attrs["NOF_DIPOLE_DATASETS"],
SELECTED_DIPOLES=f.attrs["SELECTED_DIPOLES"],
SELECTED_DIPOLES_INDEX=range(ndipoles),
CHANNEL_ID=f.attrs["CHANNEL_ID"],
TIME=[f.attrs["TIME"]] * ndipoles,
TIME_HR=[f.attrs["TIME_HR"]] * ndipoles,
SAMPLE_NUMBER=[f.attrs["SAMPLE_NUMBER"]] * ndipoles,
SLICE_NUMBER=[f.attrs["SLICE_NUMBER"]] * ndipoles,
SAMPLE_FREQUENCY=[f.attrs["SAMPLE_FREQUENCY"]] * ndipoles,
SAMPLE_FREQUENCY_UNIT=[str(f.attrs["SAMPLE_FREQUENCY_UNIT"])] *
ndipoles,
SAMPLE_FREQUENCY_VALUE=[f.attrs["SAMPLE_FREQUENCY_VALUE"]] *
ndipoles,
FREQUENCY_INTERVAL=[f.attrs["BANDWIDTH"] / self.nch] * ndipoles,
SAMPLE_INTERVAL=[f.attrs["SAMPLE_INTERVAL"]] * ndipoles,
SAMPLE_INTERVAL_UNIT=[str(f.attrs["SAMPLE_INTERVAL_UNIT"])] *
ndipoles,
DIPOLE_CALIBRATION_DELAY_UNIT=[
str(f.attrs["DIPOLE_CALIBRATION_DELAY_UNIT"])
] * ndipoles,
OBSERVER=str(f.attrs["OBSERVER"]),
# Derived keywords that we need
FREQUENCY_DATA=cr.hArray(self.channel_frequencies,
name="Frequency"),
BEAM_FREQUENCIES=cr.hArray(self.channel_frequencies,
name="Frequency"),
ALIGNMENT_REFERENCE_ANTENNA=0, # TODO: check how this is defined
PIPELINE_NAME="UNDEFINED",
BLOCK=0,
BLOCKSIZE="", # 1024
clock_offset=[
float(
md.getClockCorrectionParset(
'/home/veen/scripts/alert/StationCalibration.parset',
self.station_name, self.substation))
] * ndipoles,
MAXIMUM_READ_LENGTH=204800000, # Check
FFTSIZE=self.fftdata.shape[-1],
NYQUIST_ZONE=[2] * ndipoles,
FREQUENCY_RANGE="", # [(100000000.0, 200000000.0)] * 96
PIPELINE_VERSION="UNDEFINED",
# Derived keywords that are less relevant
TIMESERIES_DATA="", # Check
ANTENNA_POSITION="",
# [3826575.52551, 460961.8472, 5064899.465]*96
NOF_STATION_GROUPS=1, # Check
ITRF_ANTENNA_POSITION="",
# Same as ANTENNA_POSITION" but cr.hArray
CABLE_LENGTH=cr.hArray([115] * ndipoles), # Check
OBSERVATION_END_TAI="UNDEFINED",
OBSERVATION_STATION_LIST=["UNDEFINED"],
STATION_GAIN_CALIBRATION="", # No idea
LCR_ANTENNA_POSITION="", # No idea
CABLE_DELAY="", # Error
SCR_ANTENNA_POSITION="", # No idea
OBSERVATION_START_TAI="UNDEFINED",
DATA_LENGTH=[204800000] * ndipoles, # Check
CABLE_DELAY_UNIT="", # Error
# Possibly generated keywords
TIME_DATA="", # No idea
FFT_DATA=cr.hArray(self.fftdata, name="fft(E-Field)")
#"EMPTY_TIMESERIES_DATA = "",
)
# Writing to file
hfftdata.write(self.dynspecfile,
writeheader=True,
clearfile=True,
ext='beam')
# Writing header
hfftdata.writeheader(self.dynspecfile, ext='beam')
#Add beams
TAB = bt.addBeams(beams, dm=DM)
#Calculate dynamic spectrum.
dynspec, cleandynspec = bt.rawdyncalc(TAB, clean=True, tbin=tbin)
#---------------------------
#If saved binary.
elif adding_beams == 2:
filename = BEAMDATA + 'pol1.cleandynspec.bin'
cleandynspec = cr.hArray(complex, [8193L, 764L])
cleandynspec.readfilebinary(filename, 0)
elif adding_beams == 3:
filename_pol1 = BEAMDATA + beams_dir + 'L43784_D20120125T211154.Superterp.pol1.TAB.bin'
TAB_pol1 = cr.hArray(complex, [12224, 8193])
TAB_pol1.readfilebinary(filename_pol1, 0)
filename_pol0 = BEAMDATA + beams_dir + 'L43784_D20120125T211154.Superterp.pol0.TAB.bin'
TAB_pol0 = cr.hArray(complex, [12224, 8193])
TAB_pol0.readfilebinary(filename_pol0, 0)
#Calculate dynamic spectrum.
# dynspec_pol1,cleandynspec_pol1 = bt.rawdyncalc(TAB_pol1,clean=True,tbin=tbin)
dynspec_pol0, cleandynspec_pol0 = bt.rawdyncalc(TAB_pol0,
clean=True,
tbin=tbin)
cleandynspec = cleandynspec_pol0 #+ cleandynspec_pol1
cleandynspec.par = cleandynspec_pol0.par
def ccBeams(beams,
ref_station='CS002',
freq_range=[130, 160],
time_range=[0, 0.1],
verbose=False,
antenna_set='',
inv_base_fit=1):
''' Cross correlate the dedispersed pulses.
============== ========= ===================================================================
*ref_station* CS002 Reference station to which all the other will be cross correlated.
*freq_range* [130,160] List with the begin and end of frequency selection. [f1,f2] in MHz
*time_range* [0,0.1] List with the begin and end of time selection. [t1,t2] in s
*verbose* False Prints and plots releavant information.
*antenna_set* None Antenna set of the station to which all the other will be CCed, specially used for HBA1 substations.
============== ========= ===================================================================
Example::
import Beam_Tools as bt
ST_DELAYS = bt.ccBeams(beams,freq_range=[175,195],time_range=[.345,.355],verbose=1)
'''
if not antenna_set:
if 'LBA' in beams['ANTENNA_SET'][0]:
raise NotImplementedError(
'This code is optimized for HBA (Nyquist 2) FFTs. Since invfftw is bugged for Nyquist Zone 2. Also, reference antena is for HBA0.'
)
antenna_set = beams['ANTENNA_SET'][0]
factor = [0, 1, 0]
else:
antenna_set = 'HBA0'
factor = [1, 2, beams['BEAM_SPECLEN']]
#Find the reference station index
try:
i = np.arange(beams['NOF_BEAM_DATASETS'])[
(np.array(beams['STATION_NAME']) == ref_station)
& (np.array(beams['ANTENNA_SET']) == antenna_set)]
ref_station_num = int(i[0])
except:
print 'Refence station not found. Using station: ' + beams[
'STATION_NAME'][0]
ref_station_num = 0
#Frequency selection indexing.
if type(freq_range) != type([]) or len(freq_range) != 2 or freq_range[
0] > freq_range[1] or freq_range[0] < 0 or freq_range[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
freq_range = cr.hArray(float, [2], fill=freq_range,
units=('M', 'Hz')).setUnit("", "Hz")
frequencies = beams.getFrequencies()
frequency_slice = cr.hArray(int, 2)
cr.hFindSequenceBetweenOrEqual(frequency_slice, frequencies, freq_range[0],
freq_range[1], 0, 0)
#Time selection indexing.
if type(time_range) != type([]) or len(time_range) != 2 or time_range[
0] > time_range[1] or time_range[0] < 0 or time_range[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
times = beams.getTimes()
times.setUnit('', 's')
block_range = cr.hArray(int, 2)
cr.hFindSequenceBetweenOrEqual(block_range, times, time_range[0],
time_range[1], 0, 0)
nblocks = block_range[1] - block_range[0]
#Large time series matrix
ts_matrix = cr.hArray(float, (nblocks, beams['NOF_BEAM_DATASETS'],
beams['BEAM_BLOCKLEN'] * factor[1]))
t0 = time.clock()
#-----------------
#CC loop
for i in range(nblocks):
if verbose:
print 'CC calculation at {0:.2%} \r'.format(float(i) / nblocks),
sys.stdout.flush()
#Correlating frequecies only where pulse is present.
#Gaussian smothing of the frequency band.
gaussian_weights = cr.hArray(
cr.hGaussianWeights(int(beams["BLOCKSIZE"] / 100), 8.0))
bandpass_filter = cr.hArray(float, len(frequencies), fill=1)
bandpass_filter[:frequency_slice[0]] = 0
bandpass_filter[frequency_slice[1]:] = 0
cr.hRunningAverage(bandpass_filter, gaussian_weights)
# fft_matrix[...,:frequency_slice[0]] = 0+0j
# fft_matrix[...,frequency_slice[1]:] = 0+0j
#Gettind the data.
fft_matrix = beams.empty('FFT_DATA')
beams.getFFTData(fft_matrix, int(block_range[0] + i))
#For frequency gain calibration. (Temporarily here.)
# fft_matrix *= inv_base_fit
# Apply bandpass
fft_matrix[...].mul(bandpass_filter)
#Cross correlation: vec1*conj(vec2)
fft4cc = cr.hArray(complex, [1, beams['BEAM_SPECLEN']],
fft_matrix[ref_station_num])
for beam in np.arange(beams['NOF_BEAM_DATASETS'])[(np.array(
range(beams['NOF_BEAM_DATASETS'])) != ref_station_num)]:
fft_matrix[int(beam)].crosscorrelatecomplex(fft4cc, True)
fft_matrix[ref_station_num].crosscorrelatecomplex(
fft4cc, True) #Autocorrelation.
#------------------------
#There is a difference in defintion between ffts (here fftw and fftcasa).
#If the fftw definition is used, then one of the FFT data vectors still needs to be multiplied by -1,1,-1,...
#to get the peak of the crosscorrelation for lag = 0 in the middle of floatvec.
#This makes sense since the lag can be positive or negative.
fft_matrix *= -1
#------------------------
#Upsampling
ts_cc = cr.hArray(
float,
[beams['NOF_BEAM_DATASETS'], beams['BEAM_BLOCKLEN'] * factor[1]],
)
long_fft = cr.hArray(complex, [
beams['NOF_BEAM_DATASETS'],
beams['BEAM_SPECLEN'] * factor[1] - factor[0]
])
for beam in range(beams['NOF_BEAM_DATASETS']):
long_fft[beam, factor[2]:] = fft_matrix[beam]
ts_cc[beam].invfftw(long_fft[beam])
ts_matrix[i] = ts_cc
if verbose:
print "Finished - ccLoop in %f sec" % (time.clock() - t0)
CAL_DELAY = cr.hArray(float, beams['NOF_BEAM_DATASETS'])
#-----------------
#Finding time delay - loop.
for beam in range(beams['NOF_BEAM_DATASETS']):
#------------------------
#Single block CC averaging.
cross_correlation = cr.hArray(
float,
[beams['NOF_BEAM_DATASETS'], beams['BEAM_BLOCKLEN'] * factor[1]])
cc = cr.hArray(float, [1, beams['BEAM_BLOCKLEN'] * factor[1]])
for block in range(nblocks):
cc += ts_matrix[block, beam]
cross_correlation[beam] = cc / nblocks
#------------------------
# Resampling
resample = 32
delay_step = beams['TBB_SAMPLE_INTERVAL'][0] / resample
cc_smooth = cr.hArray(float, [1, beams['BEAM_BLOCKLEN'] * resample])
cr.hFFTWResample(cc_smooth, cross_correlation[beam])
#------------------------
#Fitting cc.
mx = cr.trun("FitMaxima",
cc_smooth,
peak_width=200,
splineorder=2,
doplot=verbose,
newfigure=verbose)
delay = mx.lags[0] * delay_step - delay_step * beams[
'BEAM_BLOCKLEN'] / 2 * resample
CAL_DELAY[beam] = delay
if verbose:
print '--------------------'
print 'CC of %s with %s' % (beams['FILENAMES'][beam],
beams['FILENAMES'][ref_station_num])
print 'Calibration Delay = ' + str(
mx.lags[0] * delay_step -
delay_step * beams['BEAM_BLOCKLEN'] / 2 * resample) + ' [sec]'
return CAL_DELAY
def addBeams(beams,
dyncalc=False,
save_file=False,
fraction=False,
clean=False,
tbin=1,
dm=0,
verbose=1,
incoherent=False):
'''Add complex beams toguether.
If requested:
- Calculates a dynamic spectrum.
- Dedisperses the beam.
- Bins the dynspec in integer time blocks (not for the TAB, which stays in complex).
=============== ===== ===================================================================
*beams* Input beams, opened by the beam.py interphase.
*dyncalc* True If True it calculates the dynamic spectrum
*dm* 0 Dedispersion Measure
*save_file* False Saves a file in hArray format with additional information besides the array values.
*fraction* None If not None, then a list of the form [x,y] such that extracting the fraction x/y of the data. with x>-1, and y>=x. This also makes tbin=1
*tbin* 1 If >1 integrates over this number of blocks. Or time binning.
*clean* False If True it calculates the cleaned spectrum.
*verbose* 1 If want to print status and performance.
*incoherent* False Adding beams coherently (default) or not.
=============== ===== ===================================================================
Example::
import Beam_Tools as bt
TAB,dynspec,cleandynspec = bt.addBeams(beams,dyncalc=True,tbin=16,dm=26.76,clean=True)
'''
if incoherent:
print 'Incoherent addition of the following beams: '
else:
print 'Coherent addition of the following beams: '
for file in beams['FILENAMES']:
print file
print '------_-------'
if save_file:
raise KeyError("Keyword is invalid for now: " + key)
if fraction:
if tbin > 1:
print 'WARNING: tbin has been reset to 1, since using fraction.'
tbin = 1
t0 = time.clock()
speclen = beams['BLOCKSIZE'] / 2 + 1
block_duration = beams['BLOCKSIZE'] * beams['SAMPLE_INTERVAL'][0]
nblocks = beams['NCHUNKS'] * beams['BEAM_NBLOCKS']
beam = beams.empty('FFT_DATA')
total_beam = cr.hArray(complex, (nblocks, beam.shape()[1]))
fft_matrix = cr.hArray(complex, beam.shape()[1])
if not beams['DM'] and dm:
beams['DM'] = dm
if not fraction:
fraction = [1, 1]
else:
if type(fraction) != type([]) or len(fraction) != 2 or fraction[
0] > fraction[1] or fraction[0] < 0 or fraction[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
if fraction[0] == 0: fraction[0] = 1
nblocks = int(nblocks / fraction[1])
if dyncalc:
dynspec = cr.hArray(float, (nblocks, beam.shape()[1]))
if incoherent:
for block in range(nblocks):
bm = cr.hArray(complex, beam.shape()[1])
if verbose:
print 'Addition beams at {0:.2%} \r'.format(
float(block) / nblocks),
sys.stdout.flush()
beams.getFFTData(beam, block)
dynspec[block].spectralpower2(beam[...])
total_beam = dynspec
else:
for block in range(nblocks):
bm = cr.hArray(complex, beam.shape()[1])
if verbose:
print 'Addition beams at {0:.2%} \r'.format(
float(block) / nblocks),
sys.stdout.flush()
beams.getFFTData(beam, block)
for b in range(beam.shape()[0]):
fft_matrix[:] = beam[b].vec() / float(b + 1.)
if b > 0:
bm *= (b) / (b + 1.)
bm += fft_matrix #Around here I could possibly change the code to be saving each station at the time... just like beamformer does.. to help not eating too much memory.
total_beam[block] = bm
if dyncalc:
dynspec[block].spectralpower2(bm)
if verbose:
print "Finished - addBeams in %f sec" % (time.clock() - t0)
if dyncalc:
start_time = (fraction[0] - 1) * (block_duration * nblocks)
end_time = fraction[0] * (block_duration * nblocks)
#Create a frequency vector
frequencies = beams['BEAM_FREQUENCIES']
#Create a time vector
times = cr.hArray(
float,
int(round((end_time - start_time) / (block_duration * tbin))),
name="Time",
units=("", "s"))
times.fillrange(start_time, block_duration * tbin)
#Time integration.
if tbin > 1:
dynspec = cr.hArray_toNumpy(dynspec)
dynspec = dynspec.reshape(
(dynspec.shape[0] / tbin, tbin, dynspec.shape[1]))
dynspec = np.sum(dynspec, axis=1)
dynspec = cr.hArray(dynspec)
#Cleaning dynamic spectrum.
if clean:
avspec = cr.hArray(float, speclen, fill=0.0)
dynspec[...].addto(avspec)
cleandynspec = dynspec / avspec
#Transposing arrays.
dynspec = dynspec.Transpose()
if clean:
cleandynspec = cleandynspec.Transpose()
#Adding parameters
dynspec.par.yvalues = frequencies
dynspec.par.xvalues = times
dynspec.par.tbin = tbin
if clean:
cleandynspec.par.yvalues = frequencies
cleandynspec.par.xvalues = times
cleandynspec.par.tbin = tbin
if clean:
return total_beam, dynspec, cleandynspec
else:
return total_beam, dynspec
else:
return total_beam
def cutDynspec(dynspec, freq_range=None, time_range=None, hdr=None):
'''Cuts a dynamic spectrum around the dedispersed peak.
============== ===== ===================================================================
*dynspec* Dynamic spectrum produced either by addBeams or rawdyncalc.
*freq_range* None Frequency range in MHz
*time_range* None Time range in seconds
============== ===== ===================================================================
Example::
import Beam_Tools as bt
zoom_dynspec = bt.cutDynspec(cleandynspec,freq_range=[175,195],time_range=[0.1,0.2])
'''
#-----------
#Log of things to check:
#1)tbin divisiion
#-----------
info = copy.deepcopy(dynspec.par)
dynspec = dynspec.Transpose()
nblocks = dynspec.shape()[0]
if hdr:
speclen = hdr['FFTSIZE']
blocklen = hdr['BLOCKSIZE'] * info.tbin
dt_sample = cr.asval(hdr['SAMPLE_INTERVAL'])
f0 = hdr['BEAM_FREQUENCIES'][0]
f1 = hdr['BEAM_FREQUENCIES'][-1]
df = cr.asval(hdr['FREQUENCY_INTERVAL'])
else:
speclen = 8193
blocklen = (8192) * 2
dt_sample = 5e-9
f0 = 1e+08
f1 = 2e+08
df = 12207.03125
dt = dt_sample * blocklen * info.tbin
frequencies = info.yvalues
if not freq_range:
frequency_slice = [0, dynspec.shape()[1]]
nchannels = dynspec.shape()[1]
else:
freq_range = cr.hArray(float, [2], fill=freq_range,
units=('M', 'Hz')).setUnit("", "Hz")
frequency_slice = cr.hArray(int, 2)
cr.hFindSequenceBetweenOrEqual(frequency_slice, frequencies,
freq_range[0], freq_range[1], 0, 0)
nchannels = frequency_slice[1] - frequency_slice[0]
times = info.xvalues
if not time_range:
time_slice = [0, dynspec.shape()[0]]
nblocks = dynspec.shape()[0]
else:
time_slice = cr.hArray(int, 2)
cr.hFindSequenceBetweenOrEqual(time_slice, times, time_range[0],
time_range[1], 0, 0)
nblocks = time_slice[1] - time_slice[0]
#First cutting the time axis.
sliced_dynspec = cr.hArray(float, [nblocks, speclen],
dynspec[time_slice[0]:time_slice[1]])
sliced_dynspec = sliced_dynspec.Transpose()
#Then cutting the frequency axis.
zoom_dynspec = cr.hArray(
float, [nchannels, nblocks],
sliced_dynspec[frequency_slice[0]:frequency_slice[1]])
times = cr.hArray(
float, nblocks,
list(np.arange(time_slice[0] * dt, time_slice[1] * dt, dt)))
frequencies = cr.hArray(
float, nchannels,
frequencies[frequency_slice[0]:frequency_slice[1]].vec())
zoom_dynspec.par.xvalues = times
zoom_dynspec.par.yvalues = frequencies
zoom_dynspec.par.tbin = info.tbin
return zoom_dynspec
#f["ANTENNA_SET"]='LBA_OUTER'
# Set antenna selection
f.setAntennaSelection(selection)
if not options.ntimesteps:
ntimesteps = f["MAXIMUM_READ_LENGTH"] / (nblocks * blocksize)
else:
ntimesteps = options.ntimesteps
print "Number of time steps:", ntimesteps
# Get frequencies
frequencies = f.getFrequencies()
# Create frequency mask
mask = cr.hArray(int, nfreq, fill=0)
# Get observation time (as UNIX timestamp)
obstime = f["TIME"][0]
# Set image parameters
# Around the Crab (J2000)
NAXIS = 2
NAXIS1 = 240 #options.naxis1
NAXIS2 = 240 #options.naxis2
CTYPE1 = 'ALON_SIN' #'RA---SIN'
CTYPE2 = 'ALAT_SIN' #'DEC--SIN'
CTYPE3 = 'FREQ'
CTYPE4 = 'TIME'
LONPOLE = 0.
LATPOLE = 90.
beams['DM'] = 26.76
block_range=np.arange(240,320)
nblocks = len(block_range)
blocklen = 16384
speclen = 8193
# Do mean of complex or timeseries
mean_fft =1
#pdb.set_trace()
if mean_fft:
cross_correlation = cr.hArray(float,[1,16384]) #beams.empty('TIMESERIES_DATA')[0]
fft_matrix = cr.hArray(complex,[nblocks,speclen]) # ,len(filenames)
else:
cross_correlation = cr.hArray(float,[nblocks,blocklen])
#Frequency masking
Freq_Range=[149,169]
frequencies = cr.hArray(float,speclen, list(np.arange(1e+08,2e+08+12207.03125,12207.03125)))
frequency_slice = cr.hArray(int, 2)
frequencies/=10**6
cr.hFindSequenceBetweenOrEqual(frequency_slice,frequencies,Freq_Range[0], Freq_Range[1], 0, 0)
for i,block in enumerate(block_range):
print 'Calculation at {0:.2%} \r'.format(float(block-block_range[0])/nblocks),
sys.stdout.flush()
fft = beams.empty('FFT_DATA')
'L43784_D20120125T211154.866Z_CS005_R000_tbb.pol1.multi_station.beam',
'L43784_D20120125T211154.871Z_CS006_R000_tbb.pol1.multi_station.beam',
'L43784_D20120125T211154.887Z_CS007_R000_tbb.pol1.multi_station.beam'
]
filenames = filenames[:2]
beams = cr.open(filenames)
beams['NCHUNKS'] = 191
factor = .1
delay_step = beams['TBB_SAMPLE_INTERVAL'][0] * factor
delay_range = np.arange(delay_step * 110, delay_step * 131, delay_step)
for i, dt in enumerate(delay_range):
if abs(dt) < 1e-15: #Removing unfeasible time resolution in delays
delay_range[i] = 0
SNR = cr.hArray(float, len(delay_range), fill=0)
for i, dt in enumerate(delay_range):
beams['CAL_DELAY'] = [0.0, dt]
TAB = bt.addBeams(beams, dm=26.776, verbose=interactive)
dynspec, cleandynspec = bt.rawdyncalc(TAB, clean=True, verbose=interactive)
zoom_dynspec = bt.cutdynspec(cleandynspec, pulse=0)
flat_cleandyn = bt.flatdyn(zoom_dynspec)
stats = dostats(flat_cleandyn)
SNR[i] = stats[3]
if verbose:
print '----------------------------------'
print 'Original Mean ', stats[0]
#beams['CAL_DELAY'] = [0,1.38e-9]
#print 'cal-delay', beams['CAL_DELAY']
#block_range=np.arange(3662,3906) # Where pulse 2 is located.
#block_range=np.arange(7250,7280) # Where pulse 1 is located.
block_range = np.arange(260, 300) # Where pulse 3 is located.
print 'Block range : ', block_range
nblocks = len(block_range)
blocklen = 16384
speclen = 8193
mean_at_fft = 0
envelope = 0
cross_correlation = cr.hArray(
float, [1, blocklen]) #beams.empty('TIMESERIES_DATA')[0]
fft_matrix = cr.hArray(complex, [nblocks, speclen]) # ,len(filenames)
ts_matrix = cr.hArray(complex, [nblocks, blocklen * 2])
invfftplan = cr.FFTWPlanManyDftC2r(blocklen, 1, 1, 1, 1, 1,
cr.fftw_flags.ESTIMATE)
#-----------------
#Frequency masking
#Freq_Range=[140,155] #Pulse2
Freq_Range = [149, 169] #Pulse3
frequencies = cr.hArray(
float, speclen, list(np.arange(1e+08, 2e+08 + 12207.03125, 12207.03125)))
frequency_slice = cr.hArray(int, 2)
frequencies /= 10**6
cr.hFindSequenceBetweenOrEqual(frequency_slice, frequencies, Freq_Range[0],
def dynDM(beams,
dynspec,
DM=0,
Ref_Freq=None,
from_file=False,
verbose=False,
save_file=False):
"""
Do dedispersion by integer shifting.
Calculate dedispersed time series.
============== ===== ===================================================================
*beams* Input beam
*cleandynspec* None Array with cleaned dynamic spectrum.
*DM* 0 Dispersion Measure.
*Ref_Freq* None Reference frequencies in Hz
*from_file* False Read cleandynspec from file.
*verbose* False If true then prints extra information, plot the dedispersed dynspec, and calculates/plots the time series.
*save_file* False Saves a file in hArray format with additional information besides the array values.
============== ===== ===================================================================
Example::
dedispersed_dynspec = Task.dynDM(cleandynspec=cleandynspec,DM=26.83,Ref_Freq=[151e6,170e6])
or::
Task.dynDM(dynspec,DM=26.83,Ref_Freq=170e6,from_file=True,verbose=True)
Preferably use a cleaned dynamic spectrum as input.
"""
raise NotImplementedError
Ref_Freq = cr.asval(Ref_Freq)
block_duration = beams['BLOCKSIZE'] * beams['SAMPLE_INTERVAL'][0]
tbin = cleandynspec.par.tbin
#Create a frequency vector
frequencies = cleandynspec.par.yvalues
#Create a time vector
times = cleandynspec.par.xvalues
#Dedispersion parameters
dedispersed_dynspec = cr.hArray(float, cleandynspec, fill=0)
dt = block_duration * tbin
#Calculate the relative shifts in samples per frequency channels
#Constant value comes from "Handbook of Pulsar Astronomy - by Duncan Ross Lorimer , Section 4.1.1, pagina 86 (in google books)"
shifts = (4.148808e-3 * DM / dt) * 1e9**2 * (Ref_Freq**-2 -
frequencies**-2)
#Integer offsets to reference frequency (shift to center)
offsets = cr.Vector(int, frequencies, fill=shifts)
# offsets += times.shape()[0]/2
#Now do the actual dedispersion by integer shifting ... that's all
dedispersed_dynspec[...].shift(cleandynspec[...], offsets.vec())
#Adding parameters
dedispersed_dynspec.par.yvalues = frequencies
dedispersed_dynspec.par.xvalues = times
if save_file:
dedispersed_dynspec.write(os.path.join(filename,
"dedispersed_dynspec"),
nblocks=1,
block=0,
clearfile=True)
print 'Saving binary in %s' % os.path.join(filename,
"dedispersed_dynspec.pcr")
return dedispersed_dynspec
def calibratedGaincurve(freq, NrAntennas,galaxy = True):
"""
Function delivers calibration curve as::
Data * Calibration curve = Simulated voltage
= Expected electric field * Antenna model
Hence, it's the gain-factor by which the data should be multiplied in order to match the expected voltages.
"""
if galaxy == True:
Calibration_curve = np.zeros(101)
Calibration_curve[29:82] = np.array([0, 1.37321451961e-05,
1.39846332239e-05,
1.48748993821e-05,
1.54402170354e-05,
1.60684568225e-05,
1.66241942741e-05,
1.67039066047e-05,
1.74480931848e-05,
1.80525736486e-05,
1.87066855054e-05,
1.88519099831e-05,
1.99625051386e-05,
2.01878566584e-05,
2.11573680797e-05,
2.15829455528e-05,
2.20133824866e-05,
2.23736319125e-05,
2.24484419697e-05,
2.37802483891e-05,
2.40581543111e-05,
2.42020383477e-05,
2.45305869187e-05,
2.49399905965e-05,
2.63774023804e-05,
2.70334253414e-05,
2.78034857678e-05,
3.07147991391e-05,
3.40755705892e-05,
3.67311849851e-05,
3.89987440028e-05,
3.72257913465e-05,
3.54293510934e-05,
3.35552370942e-05,
2.96529815929e-05,
2.79271252352e-05,
2.8818544973e-05,
2.92478843809e-05,
2.98454768706e-05,
3.07045462103e-05,
3.07210553534e-05,
3.16442871206e-05,
3.2304638838e-05,
3.33203882046e-05,
3.46651060935e-05,
3.55193137077e-05,
3.73919275937e-05,
3.97397037914e-05,
4.30625048727e-05,
4.74612081994e-05,
5.02345866124e-05,
5.53621848304e-05,
0])
## 30 - 80 MHz, derived from average galaxy model + electronics
else:
Calibration_curve = np.array([ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
9.99000000e-07, 9.13000000e-07, 1.05000000e-06,
1.16000000e-06, 1.14000000e-06, 1.14000000e-06,
1.13000000e-06, 1.21000000e-06, 1.25000000e-06,
1.25000000e-06, 1.21000000e-06, 1.29000000e-06,
1.31000000e-06, 1.33000000e-06, 1.24000000e-06,
1.22000000e-06, 1.32000000e-06, 1.30000000e-06,
1.24000000e-06, 1.20000000e-06, 1.23000000e-06,
1.22000000e-06, 1.50000000e-06, 1.42000000e-06,
1.41000000e-06, 1.50000000e-06, 1.63000000e-06,
1.71000000e-06, 1.83000000e-06, 2.04000000e-06,
1.96000000e-06, 1.69000000e-06, 1.51000000e-06,
1.53000000e-06, 1.41000000e-06, 1.30000000e-06,
1.40000000e-06, 1.43000000e-06, 1.46000000e-06,
1.49000000e-06, 1.57000000e-06, 1.59000000e-06,
1.68000000e-06, 1.75000000e-06, 1.81000000e-06,
1.96000000e-06, 2.20000000e-06, 2.46000000e-06,
3.01000000e-06, 3.66000000e-06, 4.68000000e-06,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00])
## 30 - 80 MHz, derived from crane calibration
Calibration_curve_interp = interp1d(np.linspace(0.e6,100e6,101), Calibration_curve, kind='linear')
Calibration_curve_interp = Calibration_curve_interp(freq)
Calibration_curve_interp = cr.hArray(Calibration_curve_interp)
Calibration_curve_interp_array = cr.hArray(np.zeros(NrAntennas * len(freq)).reshape(NrAntennas, len(freq)))
Calibration_curve_interp_array[...] = (Calibration_curve_interp)
return Calibration_curve_interp_array
def dyncalc_multibeam(filename=None,
beams=None,
nbeam=0,
save_file=False,
from_file=False,
fraction=None,
tbin=1,
clean=False):
'''
Calculates the dynamic spectrum.
=============== ===== ===================================================================
*beams* None Input array.
*nbeam* 0 Beam to work with, if ()beams has stored multiple ones.
*save_file* False Saves a file in hArray format with additional information besides the array values.
*from_file* False Read cleandynspec from file.
*fraction* None If not None, then a list of the form [x,y] such that extracting the fraction x/y of the data. with x>-1, and y>=x.
*tbin* 1 If >1 integrates over this number of blocks. Or time binning.
*clean* False If True it calculates the cleaned spectrum.
=============== ===== ===================================================================
Example::
import Beam_Tools as bt
dynspec = bt.dyncalc(filename)
or::
import Beam_Tools as bt
dynspec,cleandynspec = bt.dyncalc(beams,tbin=16,clean=True,save_file=True)
The regular and clean dynamic spectra (all blocks) are returned and stored in ``Task.dynspec`` and ``Task.cleandynspec`` respectively.
'''
raise NotImplementedError
if beams == None and filename == None:
raise ValueError(
'Need to provide either the filename or the opened .beam file')
if nbeam != 0 or from_file or save_file:
raise KeyError("Keyword is invalid for now: " + key)
t0 = time.clock()
if beams == None:
beams = cr.open(filename)
elif filename == None:
filename = beams['FILENAMES'][0]
# filename_bin = os.path.join(filename,"data.bin") #Maybe needed in the future if using "from_file" option.
speclen = beams['BLOCKSIZE'] / 2 + 1
block_duration = beams['BLOCKSIZE'] * beams['SAMPLE_INTERVAL'][0]
nblocks = beams['NCHUNKS'] * beams['BEAM_NBLOCKS']
if not fraction:
fraction = [1, 1]
else:
if type(fraction) != type([]) or len(fraction) != 2 or fraction[
0] > fraction[1] or fraction[0] < 0 or fraction[1] < 0:
raise ValueError(
'Need a list of lenght 2, with first element "<" or "=" second element. Both elements positive.'
)
if fraction[0] == 0: fraction[0] = 1
nblocks = int(nblocks / fraction[1])
start_time = (fraction[0] - 1) * (block_duration * nblocks) / fraction[1]
end_time = fraction[0] * (block_duration * nblocks) / fraction[1]
beam = beams.empty('FFT_DATA')
tm = cr.hArray(float, beam)
dynspec = cr.hArray(float, [nblocks, beam.shape()[0], speclen])
block_range = range((fraction[0] - 1) * nblocks, (fraction[0]) * nblocks)
pdb.set_trace()
#Reading beams.
for block in block_range:
print ' Calculation at {0:.2%} \r'.format(
float(block) / len(block_range)),
sys.stdout.flush()
beams.getFFTData(beam, block)
tm[...].spectralpower2(beam[...])
dynspec[block] = tm
#Time integration.
if tbin > 1:
dynspec = cr.hArray_toNumpy(dynspec)
dynspec = dynspec.reshape((dynspec.shape[0] / tbin, tbin,
dynspec.shape[1], dynspec.shape[2]))
dynspec = np.sum(dynspec, axis=1)
dynspec = cr.hArray(dynspec)
#Rearranging dimensions.
dynspec = cr.hArray_toNumpy(dynspec)
# dynspec = np.rollaxis(dynspec,0,3)
dynspec = np.swapaxes(dynspec, 0, 1)
dynspec = np.swapaxes(dynspec, 1, 2)
dynspec = cr.hArray(
dynspec.copy()) #The .copy() is quick&dirty way to avoid a bug.
#Cleaning dynamic spectrum.
if clean:
avspec = cr.hArray(float, speclen, fill=0.0)
cleandynspec = cr.hArray(float, dynspec, fill=0.0)
for dim in range(dynspec.shape()[0]):
single_dynspec = cr.hArray(float,
dynspec.shape()[1:], dynspec[dim])
single_dynspec[...].addto(avspec)
cleandynspec[dim] = single_dynspec / avspec
#Create a frequency vector
frequencies = beams['BEAM_FREQUENCIES']
#Create a time vector
times = cr.hArray(
float,
int(round((end_time - start_time) / (block_duration * tbin))),
name="Time",
units=("", "s"))
times.fillrange(start_time, block_duration * tbin)
#Adding parameters
dynspec.par.yvalues = frequencies
dynspec.par.xvalues = times
dynspec.par.tbin = tbin
if clean:
cleandynspec.par.yvalues = frequencies
cleandynspec.par.xvalues = times
cleandynspec.par.tbin = tbin
#Saving file(s).
if save_file:
dynspec.write(os.path.join(filename, "dynspec"),
nblocks=1,
block=0,
clearfile=True)
print 'Saving binary in %s' % os.path.join(filename, "dynspec.pcr")
if clean:
cleandynspec.write(os.path.join(filename, "clean_dynspec"),
nblocks=1,
block=0,
clearfile=True)
print 'Saving binary in %s' % os.path.join(filename,
"clean_dynspec.pcr")
print "Finished - total time used:", time.clock() - t0, "s."
if clean:
return dynspec, cleandynspec
else:
return dynspec
