def remove_bad_subints(infn, badsubints=None, badsubint_intervals=None):
"""Zero-weights bad subints.
The file is modified in-place. However, zero-weighting
is used for trimming, so the process is reversible.
Note: Subints are indexed starting at 0.
Inputs:
infn: name of time to remove subints from.
badchans: A list of subints to remove
badchan_intervals: A list of subint intervals
(inclusive) to remove
Outputs:
None
"""
if badsubints is None:
badsubints = config.cfg.badsubints
if badsubint_intervals is None:
badsubint_intervals = config.cfg.badsubint_intervals
zaplets = []
if badsubints:
zaplets.append("-w '%s'" % " ".join(['%d' % ww for ww in badsubints]))
if badsubint_intervals:
zaplets.extend(["-W '%d %d'" % lohi for lohi in badsubint_intervals])
if zaplets:
utils.print_info("Removing bad subints.", 2)
utils.execute("paz -m %s %s" % (" ".join(zaplets), infn.fn))
def prepare_subints(subdirs,
subints,
baseoutdir,
trimpcnt=6.25,
effix=False,
backend=None):
"""Prepare subints by
- Copying them to the temporary working directory
- De-weighting a percentage from each sub-band edge
- Converting archive format to PSRFITS
Inputs:
subdirs: List of sub-band directories containing
sub-ints to combine
subints: List of subint files to be combined.
(NOTE: These are the file name only (i.e. no path)
Each file listed should appear in each of the
subdirs.)
baseoutdir: Directory containing the sub-directories
of preprared files.
trimpcnt: Percentage (ie between 0-100) of subband
to trim from _each_ edge of the band.
(Default: 6.25%)
effix: Change observation site to eff_psrix to correct
for asterix clock offsets. (Default: False)
backend: Name of the backend. (Default: leave as is)
Outputs:
prepsubdirs: The sub-directories containing prepared files.
"""
devnull = open(os.devnull)
tmpsubdirs = []
for subdir in utils.show_progress(subdirs, width=50):
freqdir = os.path.split(os.path.abspath(subdir))[-1]
freqdir = os.path.join(baseoutdir, freqdir)
try:
os.makedirs(freqdir)
except OSError:
# Directory already exists
pass
fns = [os.path.join(subdir, fn) for fn in subints]
preproc = 'convert psrfits'
if effix:
preproc += ',edit site=eff_psrix'
if backend:
if ("," in backend) or ("=" in backend) or (' ' in backend):
raise errors.UnrecognizedValueError("Backend value (%s) is "
"invalid. It cannot "
"contain ',' or '=' or "
"' '" % backend)
preproc += ',edit be:name=%s' % backend
utils.execute(
['paz', '-j', preproc, '-E',
'%f' % trimpcnt, '-O', freqdir] + fns,
stderr=devnull)
tmpsubdirs.append(freqdir)
utils.print_info(
"Prepared %d subint fragments in %d freq sub-dirs" %
(len(subints), len(subdirs)), 3)
return tmpsubdirs
def trim_edge_channels(infn, nchan_to_trim=None, frac_to_trim=None):
"""Trim the edge channels of an input file to remove
band-pass roll-off and the effect of aliasing.
The file is modified in-place. However, zero-weighting
is used for trimming, so the process is reversible.
Inputs:
infn: name of file to trim.
nchan_to_trim: The number of channels to de-weight at
each edge of the band.
frac_to_trim: The fraction of the edge of each bad to
de-weight (a floating-point number between 0 and 0.5).
Outputs:
None
"""
if nchan_to_trim is None:
nchan_to_trim = config.cfg.nchan_to_trim
if frac_to_trim is None:
frac_to_trim = config.cfg.frac_to_trim
if nchan_to_trim > 0:
#utils.print_info("Trimming %d channels from subband edges " % \
# nchan_to_trim, 2)
numchans = int(infn['nchan'])
utils.execute('paz -m -Z "0 %d" -Z "%d %d" %s' % \
(nchan_to_trim-1, numchans-nchan_to_trim, numchans-1, infn.fn))
if frac_to_trim > 0:
#utils.print_info("Trimming %g %% from subband edges " % \
# frac_to_trim*100, 2)
utils.execute('paz -m -E %f %s' % (frac_to_trim * 100, infn.fn))
def dump_db_entries(db, obs_id, log_ids=None, file_ids=None, diag_ids=None):
dumps = []
stdout, stderr = utils.execute([
"mysqldump",
"--port=%d" % db.engine.url.port,
"--password=%s" % db.engine.url.password,
"--user=%s" % db.engine.url.username,
"--host=%s" % db.engine.url.host, db.engine.url.database, "obs",
"--where",
"obs_id=%d" % obs_id
])
dumps.append(stdout)
if log_ids:
stdout, stderr = utils.execute([
"mysqldump",
"--port=%d" % db.engine.url.port,
"--password=%s" % db.engine.url.password,
"--user=%s" % db.engine.url.username,
"--host=%s" % db.engine.url.host, db.engine.url.database, "logs",
"--where",
"log_id IN (%s)" % ",".join(["%d" % xx for xx in log_ids])
])
dumps.append(stdout)
if file_ids:
stdout, stderr = utils.execute([
"mysqldump",
"--port=%d" % db.engine.url.port,
"--password=%s" % db.engine.url.password,
"--user=%s" % db.engine.url.username,
"--host=%s" % db.engine.url.host, db.engine.url.database, "files",
"--where",
"file_id IN (%s)" % ",".join(["%d" % xx for xx in file_ids])
])
dumps.append(stdout)
if diag_ids:
stdout, stderr = utils.execute([
"mysqldump",
"--port=%d" % db.engine.url.port,
"--password=%s" % db.engine.url.password,
"--user=%s" % db.engine.url.username,
"--host=%s" % db.engine.url.host, db.engine.url.database,
"diagnostics", "--where",
"diagnostic_id IN (%s)" % ",".join(["%d" % xx for xx in diag_ids])
])
dumps.append(stdout)
return "\n".join(dumps)
def remove_bad_channels(infn,
badchans=None,
badchan_intervals=None,
badfreqs=None,
badfreq_intervals=None):
"""Zero-weight bad channels and channels containing bad
frequencies.
The file is modified in-place. However, zero-weighting
is used for trimming, so the process is reversible.
Note: Channels are indexed starting at 0.
Inputs:
infn: name of time to remove channels from.
badchans: A list of channels to remove
badchan_intervals: A list of channel intervals
(inclusive) to remove
badfreqs: A list of frequencies. The channels
containing these frequencies will be removed.
badfreq_intervals: A list of frequency ranges
to remove. The channels containing these
frequencies will be removed.
Outputs:
None
"""
if badchans is None:
badchans = config.cfg.badchans
if badchan_intervals is None:
badchan_intervals = config.cfg.badchan_intervals
if badfreqs is None:
badfreqs = config.cfg.badfreqs
if badfreq_intervals is None:
badfreq_intervals = config.cfg.badfreq_intervals
zaplets = []
if badchans:
zaplets.append("-z '%s'" % " ".join(['%d' % zz for zz in badchans]))
if badchan_intervals:
zaplets.extend(["-Z '%d %d'" % lohi for lohi in badchan_intervals])
if badfreqs:
zaplets.append("-f '%s'" % " ".join(['%f' % ff for ff in badfreqs]))
if badfreq_intervals:
zaplets.extend(["-F '%f %f'" % lohi for lohi in badfreq_intervals])
if zaplets:
utils.print_info("Removing bad channels.", 2)
utils.execute("paz -m %s %s" % (" ".join(zaplets), infn.fn))
def combine_files(rawfns):
"""Combine raw data files using psradd. The files are
blindly combined.
Intput:
rawfns: A list of data files to combine.
Output:
cmbfn: The path to the combined fully scrunched file.
"""
tmpfile, tmpfn = tempfile.mkstemp(suffix='.cmb', dir=config.tmp_directory)
os.close(tmpfile)
cmd = ['psradd', '-F', '-ip', '-P', '-j', 'DTFp', '-T', '-o', tmpfn
] + rawfns
utils.execute(cmd)
return tmpfn
def make_template(outdir, psrname, stage, rcvr, max_span=1, min_snr=0):
if os.path.isdir(outdir):
outdir = outdir
else:
raise errors.InputError("Output directory (%s) doesn't exist!" %
outdir)
filerows = list_files.get_files([psrname], stage, rcvr)
print "Found %d matching files" % len(filerows)
fns = get_files_to_combine(filerows, max_span, min_snr)
if not fns:
raise errors.TemplateGenerationError("No files for type=%s, "
"psr=%s, rcvr=%s" %
(stage, psrname, rcvr))
print "Combining %d files" % len(fns)
cmbfn = combine_files(fns)
runpaas = True
tmpdir = tempfile.mkdtemp(suffix="cg_paas", dir=config.tmp_directory)
while runpaas:
try:
print "Running paas"
utils.execute(['paas', '-D', '-i', cmbfn], dir=tmpdir)
except:
if raw_input("Failure! Give up? (y/n): ").lower()[0] == 'y':
runpaas = False
else:
if raw_input("Success! Keep template? (y/n): ").lower()[0] == 'y':
runpaas = False
outbasenm = os.path.join(outdir,
"%s_%s_%s" % (psrname, rcvr, stage))
tmpbasenm = os.path.join(tmpdir, 'paas')
shutil.copy(tmpbasenm + '.m', outbasenm + '.m')
shutil.copy(tmpbasenm + '.std', outbasenm + '.std')
shutil.copy(cmbfn, outbasenm + ".add")
# Clean up paas files
try:
shutil.rmtree(tmpdir)
except:
pass
try:
os.remove(cmbfn)
except:
pass
return outbasenm + '.std'
def correct_header(arfn,
obsinfo=None,
outfn=None,
backend='asterix',
receiver=None):
"""Correct header of asterix data in place.
Input:
arfn: The name of the input archive file.
obsinfo: A dictionary of observing log information to use.
(Default: search observing logs for matching entry)
outfn: Output file name.
(Default: same as input file name, but with .corr extension)
backend: Override backend name with this value.
(Default: asterix)
receiver: Override receiver name with this value.
(Default: Determine receiver automatically)
Output:
corrfn: The name of the corrected file.
corrstr: The parameter string of corrections used with psredit.
note: A note about header correction
"""
corrstr, note = get_correction_string(arfn,
obsinfo,
receiver=receiver,
backend=backend)
# Correct the file using 'psredit'
utils.execute(['psredit', '-e', 'corr', '-c', corrstr, arfn],
stderr=open(os.devnull))
# Assume the name of the corrected file
corrfn = os.path.splitext(arfn)[0] + ".corr"
# Confirm that our assumed file name is accurate
if not os.path.isfile(corrfn):
raise errors.HeaderCorrectionError("The corrected file (%s) does not " \
"exist!" % corrfn)
# Rename output file
if outfn is not None:
arf = utils.ArchiveFile(corrfn)
fn = outfn % arf
shutil.move(corrfn, fn)
corrfn = fn
return corrfn, corrstr, note
def prune_band(infn, response=None):
"""Prune the edges of the band. This is useful for
removing channels where there is no response.
The file is modified in-place. However, zero-weighting
is used for pruning, so the process is reversible.
Inputs:
infn: name of file to trim.
response: A tuple specifying the range of frequencies
outside of which should be de-weighted.
Outputs:
None
"""
if response is None:
response = config.cfg.rcvr_response_lims
if response is None:
utils.print_info(
"No freq range specified for band pruning. Skipping...", 2)
else:
# Use absolute value in case band is flipped (BW<0)
lofreq = infn['freq'] - np.abs(0.5 * infn['bw'])
hifreq = infn['freq'] + np.abs(0.5 * infn['bw'])
utils.print_info("Pruning frequency band to (%g-%g MHz)" % response, 2)
utils.print_debug("Archive's freq band (%g-%g MHz)" % \
(lofreq, hifreq), 'clean')
pazcmd = 'paz -m %s ' % infn.fn
runpaz = False # Only run paz if either of the following clauses are True
if response[0] > lofreq:
# Part of archive's low freqs are outside rcvr's response
pazcmd += '-F "%f %f" ' % (lofreq, response[0])
runpaz = True
if response[1] < hifreq:
# Part of archive's high freqs are outside rcvr's response
pazcmd += '-F "%f %f" ' % (response[1], hifreq)
runpaz = True
if runpaz:
utils.execute(pazcmd)
else:
warnings.warn("Not pruning band edges! All data are " \
"within the receiver's response.", \
errors.CoastGuardWarning)
def calibrate(infn, caldbpath, nchans=None):
"""Calibrate a pulsar scan using the calibrator database provided.
Inputs:
infn: The name of the archive to calibrate.
caldbpath: The path to a calibrator database to use.
nchans: Scrunch the input file to this many
channels before calibrating.
(Default: don't scrunch)
Outputs:
polcalfn: The name of the polarization calibrator used.
"""
if not os.path.isfile(caldbpath):
raise errors.DataReductionFailed("Calibrator database "
"file not found (%s)." % caldbpath)
if nchans is not None:
preproc = ['-j', 'F %d' % nchans]
else:
preproc = []
# Now calibrate, scrunching to the appropriate
# number of channels
stdout, stderr = utils.execute(['pac', '-d', caldbpath,
infn] + preproc)
# Get name of calibrator used
calfn = None
lines = stdout.split("\n")
for ii, line in enumerate(lines):
if line.strip() == "pac: PolnCalibrator constructed from:":
calfn = lines[ii+1].strip()
# Insert log message
utils.log_message("Polarization calibrator used:"
"\n %s" % calfn, 'info')
break
return calfn
def main():
psrname = utils.get_prefname(args.psrname)
if args.nchan == 1:
ext = '.FTp'
scrunchargs = ['-F']
elif args.nchan > 1:
ext = '.Tp.F%d' % args.nchan
scrunchargs = ['--setnchn', '%d' % args.nchan]
else:
raise ValueError("Cannot scrunch using negative number of "
"channels (nchan=%d)" % args.nchan)
#psrdirs = dict([(utils.get_prefname(os.path.basename(dd)),
# os.path.basename(dd))
# for dd in glob.glob(os.path.join(PARFILE_DIR, '*'))
# if os.path.isdir(dd)])
#if psrname in psrdirs:
# legacydir = os.path.join('/homes/plazarus/research/epta-legacy/',
# psrdirs[psrname])
#else:
# legacydir = None
# Copy EPTA legacy TOAs
#if legacydir and not os.path.exists("epta-legacy"):
# os.mkdir("epta-legacy")
# shutil.copytree(os.path.join(legacydir, "tims"), "epta-legacy/tims")
# shutil.copy(os.path.join(legacydir,
# "%s_t2noise.model" % psrdirs[psrname]),
# "epta-legacy")
# Find parfile
if args.parfile is not None:
if not os.path.exists(args.parfile):
raise errors.InputError("Parfile specified (%s) doesn't exist!" %
args.parfile)
inparfn = args.parfile
else:
# Create parfile
#inparfn = os.path.join('/homes/plazarus/research/epta-legacy/',
# psrdirs[psrname], "%s.par" % psrdirs[psrname])
inparfn = reduce_data.PARFILES[psrname]
#intimfn = os.path.join('/homes/plazarus/research/epta-legacy/',
# psrdirs[psrname], "%s_all.tim" % psrdirs[psrname])
outparfn = "%s.T2.par" % psrname
with open(inparfn, 'r') as inff, open(outparfn, 'w') as outff:
for line in inff:
# Don't copy over JUMPs or EFACs to 'outff'
if not line.startswith("JUMP") and \
not 'EFAC' in line:
outff.write(line)
outff.write("\n".join(EXTRA_PARFILE_LINES))
template_dir = os.path.join(BASE_TEMPLATE_DIR, psrname)
for stage in STAGES:
if stage == "current":
continue
for rcvr in RCVRS:
template_name = "%s_%s_%s.std" % (psrname, rcvr, stage)
# First, check if templates exists
if not os.path.isfile(os.path.join(template_dir, template_name)):
# Make template
utils.print_info("No template (%s) found!" % template_name, 1)
try:
os.makedirs(template_dir)
except:
pass
try:
print psrname, stage, rcvr
stdfn = make_template.make_template(
template_dir, psrname, stage, rcvr)
utils.print_info("Made template: %s" % stdfn, 1)
except errors.TemplateGenerationError:
pass
timfns = []
for stage in STAGES:
# List files to reduce
rows = list_files.get_files([psrname], stage)
print len(rows)
fns = {}
# Initialize list of file names for each receiver
for rcvr in RCVRS:
fns[rcvr] = []
for row in rows:
if row['stage'] not in ('cleaned', 'calibrated'):
continue
fn = os.path.join(row['filepath'], row['filename'])
fns[row['rcvr']].append(fn)
stagetimfn = "%s_%s.tim" % (psrname, stage)
print "Opening %s" % stagetimfn
stagetimff = open(stagetimfn, 'w')
# Create file listings and generate TOAs
for rcvr in RCVRS:
print rcvr, len(fns[rcvr])
if not fns[rcvr]:
# No files
continue
# Check for existing scrunched files
toscrunch = []
scrunchedfns = []
scrunchdir = os.path.join("scrunched", rcvr)
for fn in fns[rcvr]:
scrunchfn = os.path.join(scrunchdir,
os.path.basename(fn) + ext)
scrunchedfns.append(scrunchfn)
if not os.path.exists(scrunchfn):
toscrunch.append(fn)
# Scrunch files
try:
os.makedirs(scrunchdir)
except:
pass
print "Working on %s %s" % (rcvr, stage)
for fn in utils.show_progress(toscrunch, width=50):
# Create a copy of the file with the 'eff_psrix' site
cmd = ['psredit', '-c', 'site=eff_psrix', '-O', scrunchdir, fn]
cmd.extend(['-e', fn.split('.')[-1] + ext])
utils.execute(cmd)
arfn = os.path.join(scrunchdir, os.path.basename(fn + ext))
parfn = utils.get_norm_parfile(arfn)
# Re-install ephemeris
cmd = ['pam', '-Tp', '-E', parfn, '-m', arfn] + scrunchargs
utils.execute(cmd)
toas = []
mjds = []
for row in rows:
if row['rcvr'] != rcvr:
continue
if row['stage'] not in ('cleaned', 'calibrated'):
continue
template_name = "%s_%s_%s.std" % (psrname, rcvr, row['stage'])
template = os.path.join(template_dir, template_name)
# Generate TOAs
fn = os.path.join(scrunchdir, row['filename']) + ext
print fn
stdout, stderr = utils.execute([
"pat", "-T", "-A", "FDM", "-f", "tempo2", "-C",
"rcvr chan", "-d", "-s", template, fn
])
# Parse TOAs
toalines = stdout.split('\n')
for line in toalines:
toainfo = readers.tempo2_reader(line)
if toainfo is not None:
# Formatter expects 'file' field to be called 'rawfile'
toainfo['rawfile'] = toainfo['file']
toainfo['telescope_code'] = toainfo['telescope']
toainfo['type'] = stage
toainfo['rcvr'] = rcvr
toainfo['file_id'] = row['file_id']
toainfo['obs_id'] = row['obs_id']
toainfo['shortstage'] = row['stage'][:2].upper()
if row['stage'] == 'cleaned':
toainfo['grp'] = "%s_clean" % rcvr
else:
toainfo['grp'] = "%s_cal" % rcvr
toainfo['chan'] = toainfo['extras']['chan']
toas.append(toainfo)
mjds.append(toainfo['imjd'])
# Sort TOAs
utils.sort_by_keys(toas, ['fmjd', 'imjd'])
# Format timfile
sysflag = 'EFF.AS.%(rcvr)s.%(shortstage)s'
timlines = formatters.tempo2_formatter(toas,
flags=[
('rcvr', '%(rcvr)s'),
('type', '%(type)s'),
('grp', '%(grp)s'),
('sys', sysflag),
('obsid', '%(obs_id)d'),
('fileid',
'%(file_id)d'),
('chan', '%(chan)s')
])
mjds.sort()
#offsetmjds = sorted(TIME_OFFSETS.keys())
#inds = np.searchsorted(mjds, offsetmjds)+1
# Insert extra lines from back of list
#for ind, key in reversed(zip(inds, offsetmjds)):
# timlines[ind:ind] = ["\n"+TIME_OFFSETS[key]+"\n"]
# Write out timfile
timfn = "%s_%s_%s.tim" % (psrname, rcvr, stage)
with open(timfn, 'w') as ff:
for line in timlines:
ff.write(line + "\n")
utils.print_info("Wrote out timfile: %s" % timfn)
timfns.append(timfn)
stagetimff.write("INCLUDE %s\n" % timfn)
stagetimff.close()
#outtimfn = os.path.join("epta-legacy", os.path.basename(intimfn))
#with open(intimfn, 'r') as inff, open(outtimfn, 'w') as outff:
# for line in inff:
# outff.write(line)
# for rcvr in RCVRS:
# timfn = "%s_%s_cleaned.tim" % (psrname, rcvr)
# if os.path.exists(timfn):
# outff.write("INCLUDE ../%s\n" % timfn)
# Count TOAs
#toas = load_toa.parse_timfile(outtimfn, determine_obssystem=False)
systems = {}
#for toa in toas:
# if toa['is_bad']:
# continue
# if not 'sys' in toa['extras']:
# print toa
# else:
# nn = systems.get(toa['extras']['sys'], 0)
# systems[toa['extras']['sys']] = nn+1
outparfn = "%s.T2.par" % psrname
#outparfn2 = os.path.join("epta-legacy", os.path.basename(inparfn))
with open(inparfn, 'r') as inff, open(outparfn, 'w') as outff: #, \
#open(outparfn2, 'w') as outff2:
for line in inff:
# Don't copy over JUMPs or EFACs to 'outff'
# Copy JUMPs and EFACs to 'outff2' and fit
#if line.startswith("JUMP"):
# if "-sys" in line:
# obssys = line.split()[2]
# if systems.get(obssys, 0):
# # Observing system has TOAs
# # Replace all system jumps by 0 and set the fit flag
# outff2.write(" ".join(line.split()[:3])+" 0 1\n")
# else:
# outff2.write(line)
#elif line.startswith("T2EFAC"):
# outff2.write(line)
#elif line.startswith("NITS"):
# pass
#else:
outff.write(line)
# Remove fit-flags for 'outff2'
#outff2.write(" ".join(line.split()[:2])+'\n')
outff.write("\n".join(EXTRA_PARFILE_LINES))
#outff2.write("\n".join(["JUMP -sys EFF.AS.%s.CL 0 1" % rcvr for rcvr in RCVRS]))
#outff2.write("\nNITS 3\n")
# Create a master timfile
master_timfn = "%s_all.tim" % psrname
with open(master_timfn, 'w') as ff:
for timfn in timfns:
ff.write("INCLUDE %s\n" % timfn)
utils.print_info("Wrote out master timfile: %s" % master_timfn)
def combine_subints(subdirs, subints, parfn=None, outdir=None):
"""Combine sub-ints from various freq sub-band directories.
The input lists are as created by
'group_subband_dirs' or read-in by 'read_listing'.
Inputs:
subdirs: List of sub-band directories containing
sub-ints to combine
subints: List of subint files to be combined.
(NOTE: These are the file name only (i.e. no path)
Each file listed should appear in each of the
subdirs.)
parfn: New ephemeris to install when combining subints.
(Default: Use ephemeris in archive file's header)
outdir: Directory to output combined file.
(Default: Current working directory)
Output:
outfn: The name of the combined file.
"""
if outdir is None:
outdir = os.getcwd()
subints = sorted(subints)
tmpdir = tempfile.mkdtemp(suffix="_combine", dir=config.tmp_directory)
devnull = open(os.devnull)
try:
cmbsubints = []
# Try to normalise the archive's parfile
try:
if parfn is None:
arfn = os.path.join(subdirs[0], subints[0])
normparfn = utils.get_norm_parfile(arfn)
else:
normparfn = utils.normalise_parfile(parfn)
except errors.InputError:
# No parfile present
parargs = []
else:
parargs = ['-E', normparfn]
utils.print_info("Adding freq sub-bands for each sub-int...", 2)
for ii, subint in enumerate(utils.show_progress(subints, width=50)):
to_combine = [os.path.join(path, subint) for path in subdirs]
outfn = os.path.join(tmpdir, "combined_%s" % subint)
cmbsubints.append(outfn)
utils.execute(['psradd', '-q', '-R', '-o', outfn] + parargs +
to_combine,
stderr=devnull)
arf = utils.ArchiveFile(
os.path.join(tmpdir, "combined_%s" % subints[0]))
outfn = os.path.join(
outdir, "%s_%s_%s_%05d_%dsubints.cmb" %
(arf['name'], arf['band'], arf['yyyymmdd'], arf['secs'],
len(subints)))
utils.print_info("Combining %d sub-ints..." % len(cmbsubints), 1)
utils.execute(['psradd', '-q', '-o', outfn] + cmbsubints,
stderr=devnull)
except:
raise # Re-raise the exception
finally:
if debug.is_on('reduce'):
warnings.warn("Not cleaning up temporary directory (%s)" % tmpdir, \
errors.CoastGuardWarning)
else:
utils.print_info("Removing temporary directory (%s)" % tmpdir, 2)
shutil.rmtree(tmpdir)
return outfn
def update_caldb(db, sourcename, force=False):
"""Check for new calibrator scans. If found update the calibrator database.
Inputs:
db: A Database object.
sourcename: The name of the source to match.
(NOTE: '_R' will be removed from the sourcename, if present)
force: Forcefully update the caldb
Outputs:
caldb: The path to the updated caldb.
"""
name = utils.get_prefname(sourcename)
if name.endswith('_R'):
name = name[:-2]
# Get the caldb
caldb = get_caldb(db, name)
if caldb is None:
lastupdated = datetime.datetime.min
outdir = os.path.join(config.output_location, 'caldbs')
try:
os.makedirs(outdir)
except OSError:
# Directory already exists
pass
outfn = '%s.caldb.txt' % name.upper()
outpath = os.path.join(outdir, outfn)
insert_new = True
values = {'sourcename': name,
'caldbpath': outdir,
'caldbname': outfn}
else:
lastupdated = caldb['last_modified']
outpath = os.path.join(caldb['caldbpath'], caldb['caldbname'])
insert_new = False
values = {}
with db.transaction() as conn:
if not insert_new:
# Mark update of caldb as in-progress
update = db.caldbs.update().\
values(status='updating',
last_modified=datetime.datetime.now()).\
where(db.caldbs.c.caldb_id == caldb['caldb_id'])
conn.execute(update)
select = db.select([db.files],
from_obj=[db.files.\
outerjoin(db.obs,
onclause=db.files.c.obs_id ==
db.obs.c.obs_id)]).\
where((db.files.c.status.in_(['new', 'done'])) &
(db.files.c.stage == 'calibrated') &
(db.obs.c.obstype == 'cal') &
(db.obs.c.sourcename == ('%s_R' % name)))
results = conn.execute(select)
rows = results.fetchall()
results.close()
numnew = 0
for row in rows:
if row['added'] > lastupdated:
numnew += 1
utils.print_info("Found %d suitable calibrators for %s. "
"%d are new." %
(len(rows), name, numnew), 2)
values['numentries'] = len(rows)
try:
if numnew or force:
# Create an updated version of the calibrator database
basecaldir = os.path.join(config.output_location,
name.upper()+"_R")
utils.execute(['pac', '-w', '-u', '.pcal.T', '-k', outpath],
dir=basecaldir)
except:
#raise
values['status'] = 'failed'
if insert_new:
action = db.caldbs.insert()
else:
action = db.caldbs.update().\
values(note='%d new entries added' % numnew,
last_modifed=datetime.datetime.now()).\
where(db.caldbs.c.caldb_id == caldb['caldb_id'])
conn.execute(action, values)
else:
if insert_new:
action = db.caldbs.insert()
else:
action = db.caldbs.update().\
values(status='ready',
note='%d new entries added' % numnew,
last_modified=datetime.datetime.now()).\
where(db.caldbs.c.caldb_id == caldb['caldb_id'])
conn.execute(action, values)
return outpath