def searchone(filename, scan, paramfile, logfile, bdfdir):
""" Searches one scan of filename
filename is name of local sdm ('filename.GN' expected locally).
scan is scan number to search. if none provided, script prints all.
assumes filename is an sdm.
"""
filename = os.path.abspath(filename)
scans = ps.read_scans(filename, bdfdir=bdfdir)
if scan != 0:
d = rt.set_pipeline(filename, scan, paramfile=paramfile,
fileroot=os.path.basename(filename), logfile=logfile)
rt.pipeline(d, range(d['nsegments']))
# clean up and merge files
pc.merge_segments(filename, scan)
pc.merge_scans(os.path.dirname(filename), os.path.basename(filename), scans.keys())
else:
logger.info('Scans, Target names:')
logger.info('%s' % str([(ss, scans[ss]['source']) for ss in scans]))
logger.info('Example pipeline:')
state = rt.set_pipeline(filename, scans.popitem()[0], paramfile=paramfile,
fileroot=os.path.basename(filename), logfile=logfile)
def mergeall(filename, snrmin, snrmax, bdfdir):
""" Merge cands/noise files over all scans
Tries to find scans from filename, but will fall back to finding relevant files if it does not exist.
"""
filename = os.path.abspath(filename)
bignumber = 500
if os.path.exists(filename):
scans = ps.read_scans(filename, bdfdir=bdfdir)
scanlist = sorted(scans.keys())
else:
logger.warn('Could not find file {0}. Estimating scans from available files.'.format(filename))
filelist = glob.glob(os.path.join(os.path.dirname(filename), '*{0}_sc*pkl'.format(os.path.basename(filename))))
try:
scanlist = sorted(set([int(fn.rstrip('.pkl').split('_sc')[1].split('seg')[0]) for fn in filelist]))
except IndexError:
logger.warn('Could not parse filenames for scans. Looking over big range.')
scanlist = range(bignumber)
logger.info('Merging over scans {0}'.format(scanlist))
for scan in scanlist:
pc.merge_segments(filename, scan)
pc.merge_scans(os.path.dirname(filename), os.path.basename(filename), scanlist, snrmin=snrmin, snrmax=snrmax)
def __init__(self, sdmfile, fileroot='', workdir=''):
self.sdmfile = os.path.abspath(sdmfile)
if not fileroot:
self.fileroot = os.path.split(os.path.abspath(sdmfile))[1]
else:
self.fileroot = fileroot
if not workdir:
self.workdir = os.path.split(self.sdmfile)[0]
else:
self.workdir = workdir
self.scans = ps.read_scans(sdmfile)
self.sources = ps.read_sources(sdmfile)
self.gainscans = [
sc for sc in self.scans.keys()
if 'PHASE' in self.scans[sc]['intent']
] # get all cal fields
self.bpscans = [
sc for sc in self.scans.keys()
if 'BANDPASS' in self.scans[sc]['intent']
] # get all cal fields
self.sdm = sdmpy.SDM(self.sdmfile)
if len(self.gainstr) or len(self.bpstr):
print 'Found gaincal scans %s and bpcal scans %s.' % (self.gainstr,
self.bpstr)
self.set_fluxinfo()
def mergeall(filename, snrmin, snrmax, bdfdir):
""" Merge cands/noise files over all scans
Tries to find scans from filename, but will fall back to finding relevant files if it does not exist.
"""
filename = os.path.abspath(filename)
bignumber = 500
if os.path.exists(filename):
scans = ps.read_scans(filename, bdfdir=bdfdir)
scanlist = sorted(scans.keys())
else:
logger.warn('Could not find file {0}. Estimating scans from available files.'.format(filename))
filelist = glob.glob(os.path.join(os.path.dirname(filename), '*{0}_sc*pkl'.format(os.path.basename(filename))))
try:
scanlist = sorted(set([int(fn.rstrip('.pkl').split('_sc')[1].split('seg')[0]) for fn in filelist]))
except IndexError:
logger.warn('Could not parse filenames for scans. Looking over big range.')
scanlist = range(bignumber)
logger.info('Merging over scans {0}'.format(scanlist))
for scan in scanlist:
pc.merge_segments(filename, scan)
pc.merge_scans(os.path.dirname(filename), os.path.basename(filename), scanlist, snrmin=snrmin, snrmax=snrmax)
def searchone(filename, scan, paramfile, logfile, bdfdir):
""" Searches one scan of filename
filename is name of local sdm ('filename.GN' expected locally).
scan is scan number to search. if none provided, script prints all.
assumes filename is an sdm.
"""
filename = os.path.abspath(filename)
scans = ps.read_scans(filename, bdfdir=bdfdir)
if scan != 0:
d = rt.set_pipeline(filename, scan, paramfile=paramfile,
fileroot=os.path.basename(filename), logfile=logfile)
rt.pipeline(d, range(d['nsegments']))
# clean up and merge files
pc.merge_segments(filename, scan)
pc.merge_scans(os.path.dirname(filename), os.path.basename(filename), scans.keys())
else:
logger.info('Scans, Target names:')
logger.info('%s' % str([(ss, scans[ss]['source']) for ss in scans]))
logger.info('Example pipeline:')
state = rt.set_pipeline(filename, scans.popitem()[0], paramfile=paramfile,
fileroot=os.path.basename(filename), logfile=logfile)
def read(filename, paramfile, bdfdir, scan):
""" Simple parse and return metadata for pipeline for first scan """
filename = os.path.abspath(filename)
scans = ps.read_scans(filename, bdfdir=bdfdir)
logger.info('Scans, Target names:')
logger.info('%s' % str([(ss, scans[ss]['source']) for ss in scans]))
logger.info('Example pipeline:')
state = rt.set_pipeline(filename, scan, paramfile=paramfile, logfile=False)
def testrtpipe(filename, paramfile):
""" Function for a quick test of rtpipe and queue system
filename should have full path.
"""
sc = ps.read_scans(filename, bdfdir=default_bdfdir)
scan = sc.keys()[0]
telcalfile = rtutils.gettelcalfile(telcaldir, filename, timeout=60)
lastjob = rtutils.search('default', filename, paramfile, '', [scan], telcalfile=telcalfile, redishost=redishost, bdfdir=default_bdfdir)
return lastjob
def read(filename, paramfile, bdfdir, scan):
""" Simple parse and return metadata for pipeline for first scan """
filename = os.path.abspath(filename)
scans = ps.read_scans(filename, bdfdir=bdfdir)
logger.info('Scans, Target names:')
logger.info('%s' % str([(ss, scans[ss]['source']) for ss in scans]))
logger.info('Example pipeline:')
state = rt.set_pipeline(filename, scan, paramfile=paramfile, logfile=False)
def slowms(filename, slow, redishost, bdfdir):
""" Take SDM filename and create MS with integration timescale of slow for all scans.
Queues to 'slow' queue managed by redishost.
"""
import rtpipe.parsesdm as ps
sc = ps.read_scans(filename, bdfdir=bdfdir)
logger.info('Creating measurement set for %s, scans %s' % (filename, sc.keys()))
rtutils.linkbdfs(filename, sc, bdfdir)
# Submit slow-processing job to our alternate queue.
allscanstr = ','.join(str(s) for s in sc.keys())
rtutils.integrate(filename, allscanstr, slow, redishost)
def __init__(self, sdmfile, fileroot='', workdir=''):
self.sdmfile = os.path.abspath(sdmfile)
if not fileroot:
self.fileroot = os.path.split(os.path.abspath(sdmfile))[1]
else:
self.fileroot = fileroot
if not workdir:
self.workdir = os.path.split(self.sdmfile)[0]
else:
self.workdir = workdir
self.scans = ps.read_scans(sdmfile)
self.sources = ps.read_sources(sdmfile)
self.gainscans = [sc for sc in self.scans.keys() if 'PHASE' in self.scans[sc]['intent']] # get all cal fields
self.bpscans = [sc for sc in self.scans.keys() if 'BANDPASS' in self.scans[sc]['intent']] # get all cal fields
self.sdm = sdmpy.SDM(self.sdmfile)
if len(self.gainstr) or len(self.bpstr):
print 'Found gaincal scans %s and bpcal scans %s.' % (self.gainstr, self.bpstr)
self.set_fluxinfo()
