def main(pool):
log("Hello, world.")
numWorkers = totalrank - 1
lastOffset = int(nFramesFile / (numWorkers * nChunks * nFramesAvg))
log("fileSize %d" % os.path.getsize(filename))
log("nFramesFile %d" % nFramesFile)
log("lastOffset %d" % lastOffset)
# Generate tasks (offsets in data file)
tasks = [
nChunks * nFramesAvg * (1 * numWorkers * offset_i + r)
for offset_i in xrange(lastOffset) for r in xrange(numWorkers)
]
print "TASKS:", tasks[:5], "... "
pool.map(worker, tasks)
pool.close()
def progress(t_start, fraction, message):
if fraction > 0:
elapsed = (datetime.now() - t_start).total_seconds()
remaining = timedelta(seconds=(elapsed / fraction) - elapsed)
log("%s (%0.1f%% complete, %s remaining)" %
(message, 100 * fraction, humanize.naturaldelta(remaining)))
# actually lower. The HDF file (like the DRX as well, apparently)
# can contain them in either order.
pol = 1
DMstart = 300 # Initial DM trial
DMend = 400 # Final DM trial
maxpw = 5 # Maximum pulse width to search (seconds)
thresh = 5.0 #SNR cut off
import h5py
import humanize
h5f = h5py.File(sys.argv[1], 'r')['Observation1']
num_time_bins = h5f['time'].shape[0]
tInt = h5f['time'][1] - h5f['time'][0] # Temporal resolution
log("%s time bins, %.9f sec each" %
(humanize.intcomma(num_time_bins), tInt))
# These are time offsets rather than independent filenames
time_bins_per_file = min(3000, num_time_bins / totalrank)
fn = range(0, num_time_bins, time_bins_per_file)
fpp = len(fn) / totalrank # Files per process
numberofFiles = fpp * totalrank
npws = int(np.round(np.log2(maxpw / tInt))) + 1
spectarray = np.zeros((fpp, time_bins_per_file, fch - fcl))
h5t = h5f['Tuning%d' % (pol + 1)]
freq = h5t['freq'][fcl:fch]
freq /= 10**6
cent_freq = np.median(freq)
def worker(offset):
log("Working on offset %d" % offset)
# Build the DRX file
try:
fh = open(filename, "rb")
except:
log("File not fonud: %s" % filename)
sys.exit(1)
try:
junkFrame = drx.readFrame(fh)
try:
srate = junkFrame.getSampleRate()
pass
except ZeroDivisionError:
log('zero division error')
return
except errors.syncError:
log('assuming the srate is 19.6 MHz')
srate = 19600000.0
fh.seek(-drx.FrameSize + 1, 1)
fh.seek(-drx.FrameSize, 1)
beam, tune, pol = junkFrame.parseID()
beams = drx.getBeamCount(fh)
tunepols = drx.getFramesPerObs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
if offset != 0:
fh.seek(offset * drx.FrameSize, 1)
#if nChunks == 0:
# nChunks = 1
nFrames = nFramesAvg * nChunks
centralFreq1 = 0.0
centralFreq2 = 0.0
for i in xrange(4):
junkFrame = drx.readFrame(fh)
b, t, p = junkFrame.parseID()
if p == 0 and t == 0:
try:
centralFreq1 = junkFrame.getCentralFreq()
except AttributeError:
from dp import fS
centralFreq1 = fS * ((junkFrame.data.flags[0] >> 32) &
(2**32 - 1)) / 2**32
elif p == 0 and t == 2:
try:
centralFreq2 = junkFrame.getCentralFreq()
except AttributeError:
from dp import fS
centralFreq2 = fS * ((junkFrame.data.flags[0] >> 32) &
(2**32 - 1)) / 2**32
else:
pass
fh.seek(-4 * drx.FrameSize, 1)
# Sanity check
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time + offset is greater than file length")
# Master loop over all of the file chunks
#freq = numpy.fft.fftshift(numpy.fft.fftfreq(LFFT, d = 1.0/srate))
#tInt = 1.0*LFFT/srate
#print 'Temporal resl = ',tInt
#print 'Channel width = ',1./tInt
#freq1 = freq+centralFreq1
#freq2 = freq+centralFreq2
#print tInt,freq1.mean(),freq2.mean()
masterSpectra = numpy.zeros(
(nChunks, 2, Lfch - Lfcl)) # Add 2 out front for two pols
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (nFramesAvg),
# only deal with that chunk
framesRemaining = nFrames - i * nFramesAvg
if framesRemaining > nFramesAvg:
framesWork = nFramesAvg
else:
framesWork = framesRemaining
#if framesRemaining%(nFrames/10)==0:
# print "Working on chunk %i, %i frames remaining" % (i, framesRemaining)
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((4, framesWork * 4096 / beampols),
dtype=numpy.csingle)
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < LFFT:
log('data.shape[1]< LFFT, break')
return
# Inner loop that actually reads the frames into the data array
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.readFrame(fh, Verbose=False)
except errors.eofError:
log("EOF Error")
return
except errors.syncError:
log("Sync Error")
return
beam, tune, pol = cFrame.parseID()
if tune == 0:
tune += 1
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.data.iq
count[aStand] += 1
# Calculate the spectra for this block of data, in the unit of intensity
masterSpectra[i, 0, :] = (
(numpy.fft.fftshift(numpy.abs(numpy.fft.fft2(
data[:2, :]))[:, 1:])[:, Lfcl:Lfch])**2.).mean(0) / LFFT / 2.
masterSpectra[i, 1, :] = (
(numpy.fft.fftshift(numpy.abs(numpy.fft.fft2(
data[2:, :]))[:, 1:])[:, Hfcl:Hfch])**2.).mean(0) / LFFT / 2.
# Save the results to the various master arrays
outname = "%s_%i_fft_offset_%.9i_frames" % (filename, beam, offset)
log("Writing %s" % outname)
numpy.save(outname, masterSpectra)
def main(args):
log("Hello, world.") # Mic check
nChunks = 10000 #the temporal shape of a file.
LFFT = 4096 #Length of the FFT.4096 is the size of a frame readed.
nFramesAvg = 1 * 4 * LFFT / 4096 # the intergration time under LFFT, 4 = beampols = 2X + 2Y (high and low tunes)
filename = args[0]
nFramesFile = os.path.getsize(
filename) / drx.FrameSize #drx.FrameSize = 4128
lastOffset = int(nFramesFile / (totalrank * nChunks * nFramesAvg))
log("fileSize %d" % os.path.getsize(filename))
log("nFramesFile %d" % nFramesFile)
log("lastOffset %d" % lastOffset)
#for offset in [19320000, 19520000, 19720000, 19920000]:
for offset_i in xrange(
lastOffset): # one offset = nChunks*nFramesAvg skiped
offset_i = 1. * totalrank * offset_i + rank
offset = nChunks * nFramesAvg * offset_i
log("Working on offset %d" % offset)
# Build the DRX file
try:
fh = open(filename, "rb")
except:
log("File not found: %s" % filename)
sys.exit(1)
try:
junkFrame = drx.readFrame(fh)
try:
srate = junkFrame.getSampleRate()
pass
except ZeroDivisionError:
log('zero division error')
break
except errors.syncError:
log('assuming the srate is 19.6 MHz')
srate = 19600000.0
fh.seek(-drx.FrameSize + 1, 1)
fh.seek(-drx.FrameSize, 1)
beam, tune, pol = junkFrame.parseID()
beams = drx.getBeamCount(fh)
tunepols = drx.getFramesPerObs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
if offset != 0:
fh.seek(offset * drx.FrameSize, 1)
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
centralFreq1 = 0.0
centralFreq2 = 0.0
for i in xrange(4):
junkFrame = drx.readFrame(fh)
b, t, p = junkFrame.parseID()
if p == 0 and t == 0:
try:
centralFreq1 = junkFrame.getCentralFreq()
except AttributeError:
from dp import fS
centralFreq1 = fS * ((junkFrame.data.flags[0] >> 32) &
(2**32 - 1)) / 2**32
elif p == 0 and t == 2:
try:
centralFreq2 = junkFrame.getCentralFreq()
except AttributeError:
from dp import fS
centralFreq2 = fS * ((junkFrame.data.flags[0] >> 32) &
(2**32 - 1)) / 2**32
else:
pass
fh.seek(-4 * drx.FrameSize, 1)
# Sanity check
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time + offset is greater than file length"
)
# Master loop over all of the file chunks
freq = numpy.fft.fftshift(numpy.fft.fftfreq(LFFT, d=1.0 / srate))
tInt = 1.0 * LFFT / srate
log('Temporal resl = %f' % tInt)
log('Channel width = %f' % (1. / tInt))
freq1 = freq + centralFreq1
freq2 = freq + centralFreq2
#print tInt,freq1.mean(),freq2.mean()
masterSpectra = numpy.zeros((nChunks, 2, LFFT - 1))
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (nFramesAvg),
# only deal with that chunk
framesRemaining = nFrames - i * nFramesAvg
if framesRemaining > nFramesAvg:
framesWork = nFramesAvg
else:
framesWork = framesRemaining
#if framesRemaining%(nFrames/10)==0:
# print "Working on chunk %i, %i frames remaining" % (i, framesRemaining)
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((4, framesWork * 4096 / beampols),
dtype=numpy.csingle)
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < LFFT:
log('data.shape[1]< LFFT, break')
break
# Inner loop that actually reads the frames into the data array
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.readFrame(fh, Verbose=False)
except errors.eofError:
log("EOF Error")
break
except errors.syncError:
log("Sync Error")
continue
beam, tune, pol = cFrame.parseID()
if tune == 0:
tune += 1
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.data.iq
count[aStand] += 1
# Calculate the spectra for this block of data
masterSpectra[i,
0, :] = ((numpy.fft.fftshift(
numpy.abs(numpy.fft.fft2(data[:2, :]))[:, 1:]))**
2.).mean(0) / LFFT / 2. #in unit of energy
masterSpectra[i,
1, :] = ((numpy.fft.fftshift(
numpy.abs(numpy.fft.fft2(data[2:, :]))[:, 1:]))**
2.).mean(0) / LFFT / 2. #in unit of energy
# Save the results to the various master arrays
#print masterSpectra.shape
#numpy.save('data',data)
#sys.exit()
#if i % 100 ==1 :
# print i, ' / ', nChunks
outname = "%s_%i_fft_offset_%.9i_frames" % (filename, beam, offset)
log("Writing %s" % outname)
numpy.save('waterfall' + outname, masterSpectra.mean(0))
pol = 1 # 0 = lower tunning, 1 = higher tunning.
DMstart = 0 #1.0 #initial DM trial
DMend = 5000 #90.0 #finial DM trial
npws = int(np.round(np.log2(
maxpw / tInt))) + 1 # +1 Due to in range(y) it goes to y-1 only
spect = np.load(fn[0], mmap_mode='r')[:, :, fcl:fch]
spectarray = np.zeros(
(fpp, spect.shape[0],
spect.shape[2])) # X and Y are merged already after bandpass
#cobimed spectrogram and remove background
for i in range(fpp):
log('1' + str((np.load(fn[rank * fpp + i])[:, pol, fcl:fch]).shape))
log('2' +
str(massagesp(np.load(fn[rank * fpp + i])[:, pol, fcl:fch]).shape))
spectarray[i, :, :] = massagesp(
np.load(fn[rank * fpp + i])[:, pol, fcl:fch], 10, 50)
outname = 'spectarray%.2i' % rank
log("Writing %s" % outname)
np.save(outname, spectarray)
#sys.exit()
if pol < 4:
if pol == 0:
freq = np.load('freq1.npy')[fcl:fch]
else:
freq = np.load('freq2.npy')[fcl:fch]
def main(args):
log("Hello, world.")
windownumber = 4
nChunks = 3000 #the temporal shape of a file.
#Low tuning frequency range
Lfcl = 1000 * windownumber
Lfch = 1350 * windownumber
#High tuning frequency range
Hfcl = 1825 * windownumber
Hfch = 2175 * windownumber
LFFT = 4096 * windownumber #Length of the FFT.4096 is the size of a frame readed.
nFramesAvg = 1 * 4 * windownumber # the intergration time under LFFT, 4 = beampols = 2X + 2Y (high and low tunes)
filename = args[0]
nFramesFile = os.path.getsize(
filename) / drx.FrameSize #drx.FrameSize = 4128
log("fileSize %d" % os.path.getsize(filename))
log("nFramesFile %d" % nFramesFile)
fn = sorted(glob.glob('05*.npy'))
j = numpy.zeros((len(fn)))
for i in range(len(fn)):
j[i] = fn[i][30:39]
#x = total perfect offset
x = numpy.arange(j[-1] / nChunks / nFramesAvg) * nChunks * nFramesAvg
# k = the different between perfect and real
k = numpy.setdiff1d(x, j)
for m in xrange(len(k) / totalrank):
offset = k[m * totalrank + rank]
# Build the DRX file
try:
fh = open(filename, "rb")
except:
log("File not fonud: %s" % filename)
sys.exit(1)
try:
junkFrame = drx.readFrame(fh)
try:
srate = junkFrame.getSampleRate()
pass
except ZeroDivisionError:
log('zero division error')
break
except errors.syncError:
log('assuming the srate is 19.6 MHz')
srate = 19600000.0
fh.seek(-drx.FrameSize + 1, 1)
fh.seek(-drx.FrameSize, 1)
beam, tune, pol = junkFrame.parseID()
beams = drx.getBeamCount(fh)
tunepols = drx.getFramesPerObs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
if offset != 0:
fh.seek(offset * drx.FrameSize, 1)
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
centralFreq1 = 0.0
centralFreq2 = 0.0
for i in xrange(4):
junkFrame = drx.readFrame(fh)
b, t, p = junkFrame.parseID()
if p == 0 and t == 0:
try:
centralFreq1 = junkFrame.getCentralFreq()
except AttributeError:
from dp import fS
centralFreq1 = fS * ((junkFrame.data.flags[0] >> 32) &
(2**32 - 1)) / 2**32
elif p == 0 and t == 2:
try:
centralFreq2 = junkFrame.getCentralFreq()
except AttributeError:
from dp import fS
centralFreq2 = fS * ((junkFrame.data.flags[0] >> 32) &
(2**32 - 1)) / 2**32
else:
pass
fh.seek(-4 * drx.FrameSize, 1)
# Sanity check
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time + offset is greater than file length"
)
masterSpectra = numpy.zeros((nChunks, 2, Lfch - Lfcl))
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (nFramesAvg),
# only deal with that chunk
framesRemaining = nFrames - i * nFramesAvg
if framesRemaining > nFramesAvg:
framesWork = nFramesAvg
else:
framesWork = framesRemaining
#if framesRemaining%(nFrames/10)==0:
# print "Working on chunk %i, %i frames remaining" % (i, framesRemaining)
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((4, framesWork * 4096 / beampols),
dtype=numpy.csingle)
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < LFFT:
log('data.shape[1]< LFFT, break')
break
# Inner loop that actually reads the frames into the data array
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.readFrame(fh, Verbose=False)
except errors.eofError:
log("EOF Error")
break
except errors.syncError:
log("Sync Error")
continue
beam, tune, pol = cFrame.parseID()
if tune == 0:
tune += 1
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.data.iq
count[aStand] += 1
# Calculate the spectra for this block of data, in the unit of intensity
masterSpectra[i, 0, :] = ((numpy.fft.fftshift(
numpy.abs(numpy.fft.fft2(data[:2, :]))[:, 1:])[:, Lfcl:Lfch])**
2.).mean(0) / LFFT / 2.
masterSpectra[i, 1, :] = ((numpy.fft.fftshift(
numpy.abs(numpy.fft.fft2(data[2:, :]))[:, 1:])[:, Hfcl:Hfch])**
2.).mean(0) / LFFT / 2.
# Save the results to the various master arrays
outname = "%s_%i_fft_offset_%.9i_frames" % (filename, beam, offset)
log("Writing %s" % outname)
numpy.save(outname, masterSpectra)