def remove_bad_subints(infn, badsubints=None, badsubint_intervals=None): """Zero-weights bad subints. The file is modified in-place. However, zero-weighting is used for trimming, so the process is reversible. Note: Subints are indexed starting at 0. Inputs: infn: name of time to remove subints from. badchans: A list of subints to remove badchan_intervals: A list of subint intervals (inclusive) to remove Outputs: None """ if badsubints is None: badsubints = config.cfg.badsubints if badsubint_intervals is None: badsubint_intervals = config.cfg.badsubint_intervals zaplets = [] if badsubints: zaplets.append("-w '%s'" % " ".join(['%d' % ww for ww in badsubints])) if badsubint_intervals: zaplets.extend(["-W '%d %d'" % lohi for lohi in badsubint_intervals]) if zaplets: utils.print_info("Removing bad subints.", 2) utils.execute("paz -m %s %s" % (" ".join(zaplets), infn.fn))
def prepare_subints(subdirs, subints, baseoutdir, trimpcnt=6.25, effix=False, backend=None): """Prepare subints by - Copying them to the temporary working directory - De-weighting a percentage from each sub-band edge - Converting archive format to PSRFITS Inputs: subdirs: List of sub-band directories containing sub-ints to combine subints: List of subint files to be combined. (NOTE: These are the file name only (i.e. no path) Each file listed should appear in each of the subdirs.) baseoutdir: Directory containing the sub-directories of preprared files. trimpcnt: Percentage (ie between 0-100) of subband to trim from _each_ edge of the band. (Default: 6.25%) effix: Change observation site to eff_psrix to correct for asterix clock offsets. (Default: False) backend: Name of the backend. (Default: leave as is) Outputs: prepsubdirs: The sub-directories containing prepared files. """ devnull = open(os.devnull) tmpsubdirs = [] for subdir in utils.show_progress(subdirs, width=50): freqdir = os.path.split(os.path.abspath(subdir))[-1] freqdir = os.path.join(baseoutdir, freqdir) try: os.makedirs(freqdir) except OSError: # Directory already exists pass fns = [os.path.join(subdir, fn) for fn in subints] preproc = 'convert psrfits' if effix: preproc += ',edit site=eff_psrix' if backend: if ("," in backend) or ("=" in backend) or (' ' in backend): raise errors.UnrecognizedValueError("Backend value (%s) is " "invalid. It cannot " "contain ',' or '=' or " "' '" % backend) preproc += ',edit be:name=%s' % backend utils.execute( ['paz', '-j', preproc, '-E', '%f' % trimpcnt, '-O', freqdir] + fns, stderr=devnull) tmpsubdirs.append(freqdir) utils.print_info( "Prepared %d subint fragments in %d freq sub-dirs" % (len(subints), len(subdirs)), 3) return tmpsubdirs
def trim_edge_channels(infn, nchan_to_trim=None, frac_to_trim=None): """Trim the edge channels of an input file to remove band-pass roll-off and the effect of aliasing. The file is modified in-place. However, zero-weighting is used for trimming, so the process is reversible. Inputs: infn: name of file to trim. nchan_to_trim: The number of channels to de-weight at each edge of the band. frac_to_trim: The fraction of the edge of each bad to de-weight (a floating-point number between 0 and 0.5). Outputs: None """ if nchan_to_trim is None: nchan_to_trim = config.cfg.nchan_to_trim if frac_to_trim is None: frac_to_trim = config.cfg.frac_to_trim if nchan_to_trim > 0: #utils.print_info("Trimming %d channels from subband edges " % \ # nchan_to_trim, 2) numchans = int(infn['nchan']) utils.execute('paz -m -Z "0 %d" -Z "%d %d" %s' % \ (nchan_to_trim-1, numchans-nchan_to_trim, numchans-1, infn.fn)) if frac_to_trim > 0: #utils.print_info("Trimming %g %% from subband edges " % \ # frac_to_trim*100, 2) utils.execute('paz -m -E %f %s' % (frac_to_trim * 100, infn.fn))
def dump_db_entries(db, obs_id, log_ids=None, file_ids=None, diag_ids=None): dumps = [] stdout, stderr = utils.execute([ "mysqldump", "--port=%d" % db.engine.url.port, "--password=%s" % db.engine.url.password, "--user=%s" % db.engine.url.username, "--host=%s" % db.engine.url.host, db.engine.url.database, "obs", "--where", "obs_id=%d" % obs_id ]) dumps.append(stdout) if log_ids: stdout, stderr = utils.execute([ "mysqldump", "--port=%d" % db.engine.url.port, "--password=%s" % db.engine.url.password, "--user=%s" % db.engine.url.username, "--host=%s" % db.engine.url.host, db.engine.url.database, "logs", "--where", "log_id IN (%s)" % ",".join(["%d" % xx for xx in log_ids]) ]) dumps.append(stdout) if file_ids: stdout, stderr = utils.execute([ "mysqldump", "--port=%d" % db.engine.url.port, "--password=%s" % db.engine.url.password, "--user=%s" % db.engine.url.username, "--host=%s" % db.engine.url.host, db.engine.url.database, "files", "--where", "file_id IN (%s)" % ",".join(["%d" % xx for xx in file_ids]) ]) dumps.append(stdout) if diag_ids: stdout, stderr = utils.execute([ "mysqldump", "--port=%d" % db.engine.url.port, "--password=%s" % db.engine.url.password, "--user=%s" % db.engine.url.username, "--host=%s" % db.engine.url.host, db.engine.url.database, "diagnostics", "--where", "diagnostic_id IN (%s)" % ",".join(["%d" % xx for xx in diag_ids]) ]) dumps.append(stdout) return "\n".join(dumps)
def remove_bad_channels(infn, badchans=None, badchan_intervals=None, badfreqs=None, badfreq_intervals=None): """Zero-weight bad channels and channels containing bad frequencies. The file is modified in-place. However, zero-weighting is used for trimming, so the process is reversible. Note: Channels are indexed starting at 0. Inputs: infn: name of time to remove channels from. badchans: A list of channels to remove badchan_intervals: A list of channel intervals (inclusive) to remove badfreqs: A list of frequencies. The channels containing these frequencies will be removed. badfreq_intervals: A list of frequency ranges to remove. The channels containing these frequencies will be removed. Outputs: None """ if badchans is None: badchans = config.cfg.badchans if badchan_intervals is None: badchan_intervals = config.cfg.badchan_intervals if badfreqs is None: badfreqs = config.cfg.badfreqs if badfreq_intervals is None: badfreq_intervals = config.cfg.badfreq_intervals zaplets = [] if badchans: zaplets.append("-z '%s'" % " ".join(['%d' % zz for zz in badchans])) if badchan_intervals: zaplets.extend(["-Z '%d %d'" % lohi for lohi in badchan_intervals]) if badfreqs: zaplets.append("-f '%s'" % " ".join(['%f' % ff for ff in badfreqs])) if badfreq_intervals: zaplets.extend(["-F '%f %f'" % lohi for lohi in badfreq_intervals]) if zaplets: utils.print_info("Removing bad channels.", 2) utils.execute("paz -m %s %s" % (" ".join(zaplets), infn.fn))
def combine_files(rawfns): """Combine raw data files using psradd. The files are blindly combined. Intput: rawfns: A list of data files to combine. Output: cmbfn: The path to the combined fully scrunched file. """ tmpfile, tmpfn = tempfile.mkstemp(suffix='.cmb', dir=config.tmp_directory) os.close(tmpfile) cmd = ['psradd', '-F', '-ip', '-P', '-j', 'DTFp', '-T', '-o', tmpfn ] + rawfns utils.execute(cmd) return tmpfn
def make_template(outdir, psrname, stage, rcvr, max_span=1, min_snr=0): if os.path.isdir(outdir): outdir = outdir else: raise errors.InputError("Output directory (%s) doesn't exist!" % outdir) filerows = list_files.get_files([psrname], stage, rcvr) print "Found %d matching files" % len(filerows) fns = get_files_to_combine(filerows, max_span, min_snr) if not fns: raise errors.TemplateGenerationError("No files for type=%s, " "psr=%s, rcvr=%s" % (stage, psrname, rcvr)) print "Combining %d files" % len(fns) cmbfn = combine_files(fns) runpaas = True tmpdir = tempfile.mkdtemp(suffix="cg_paas", dir=config.tmp_directory) while runpaas: try: print "Running paas" utils.execute(['paas', '-D', '-i', cmbfn], dir=tmpdir) except: if raw_input("Failure! Give up? (y/n): ").lower()[0] == 'y': runpaas = False else: if raw_input("Success! Keep template? (y/n): ").lower()[0] == 'y': runpaas = False outbasenm = os.path.join(outdir, "%s_%s_%s" % (psrname, rcvr, stage)) tmpbasenm = os.path.join(tmpdir, 'paas') shutil.copy(tmpbasenm + '.m', outbasenm + '.m') shutil.copy(tmpbasenm + '.std', outbasenm + '.std') shutil.copy(cmbfn, outbasenm + ".add") # Clean up paas files try: shutil.rmtree(tmpdir) except: pass try: os.remove(cmbfn) except: pass return outbasenm + '.std'
def correct_header(arfn, obsinfo=None, outfn=None, backend='asterix', receiver=None): """Correct header of asterix data in place. Input: arfn: The name of the input archive file. obsinfo: A dictionary of observing log information to use. (Default: search observing logs for matching entry) outfn: Output file name. (Default: same as input file name, but with .corr extension) backend: Override backend name with this value. (Default: asterix) receiver: Override receiver name with this value. (Default: Determine receiver automatically) Output: corrfn: The name of the corrected file. corrstr: The parameter string of corrections used with psredit. note: A note about header correction """ corrstr, note = get_correction_string(arfn, obsinfo, receiver=receiver, backend=backend) # Correct the file using 'psredit' utils.execute(['psredit', '-e', 'corr', '-c', corrstr, arfn], stderr=open(os.devnull)) # Assume the name of the corrected file corrfn = os.path.splitext(arfn)[0] + ".corr" # Confirm that our assumed file name is accurate if not os.path.isfile(corrfn): raise errors.HeaderCorrectionError("The corrected file (%s) does not " \ "exist!" % corrfn) # Rename output file if outfn is not None: arf = utils.ArchiveFile(corrfn) fn = outfn % arf shutil.move(corrfn, fn) corrfn = fn return corrfn, corrstr, note
def prune_band(infn, response=None): """Prune the edges of the band. This is useful for removing channels where there is no response. The file is modified in-place. However, zero-weighting is used for pruning, so the process is reversible. Inputs: infn: name of file to trim. response: A tuple specifying the range of frequencies outside of which should be de-weighted. Outputs: None """ if response is None: response = config.cfg.rcvr_response_lims if response is None: utils.print_info( "No freq range specified for band pruning. Skipping...", 2) else: # Use absolute value in case band is flipped (BW<0) lofreq = infn['freq'] - np.abs(0.5 * infn['bw']) hifreq = infn['freq'] + np.abs(0.5 * infn['bw']) utils.print_info("Pruning frequency band to (%g-%g MHz)" % response, 2) utils.print_debug("Archive's freq band (%g-%g MHz)" % \ (lofreq, hifreq), 'clean') pazcmd = 'paz -m %s ' % infn.fn runpaz = False # Only run paz if either of the following clauses are True if response[0] > lofreq: # Part of archive's low freqs are outside rcvr's response pazcmd += '-F "%f %f" ' % (lofreq, response[0]) runpaz = True if response[1] < hifreq: # Part of archive's high freqs are outside rcvr's response pazcmd += '-F "%f %f" ' % (response[1], hifreq) runpaz = True if runpaz: utils.execute(pazcmd) else: warnings.warn("Not pruning band edges! All data are " \ "within the receiver's response.", \ errors.CoastGuardWarning)
def calibrate(infn, caldbpath, nchans=None): """Calibrate a pulsar scan using the calibrator database provided. Inputs: infn: The name of the archive to calibrate. caldbpath: The path to a calibrator database to use. nchans: Scrunch the input file to this many channels before calibrating. (Default: don't scrunch) Outputs: polcalfn: The name of the polarization calibrator used. """ if not os.path.isfile(caldbpath): raise errors.DataReductionFailed("Calibrator database " "file not found (%s)." % caldbpath) if nchans is not None: preproc = ['-j', 'F %d' % nchans] else: preproc = [] # Now calibrate, scrunching to the appropriate # number of channels stdout, stderr = utils.execute(['pac', '-d', caldbpath, infn] + preproc) # Get name of calibrator used calfn = None lines = stdout.split("\n") for ii, line in enumerate(lines): if line.strip() == "pac: PolnCalibrator constructed from:": calfn = lines[ii+1].strip() # Insert log message utils.log_message("Polarization calibrator used:" "\n %s" % calfn, 'info') break return calfn
def main(): psrname = utils.get_prefname(args.psrname) if args.nchan == 1: ext = '.FTp' scrunchargs = ['-F'] elif args.nchan > 1: ext = '.Tp.F%d' % args.nchan scrunchargs = ['--setnchn', '%d' % args.nchan] else: raise ValueError("Cannot scrunch using negative number of " "channels (nchan=%d)" % args.nchan) #psrdirs = dict([(utils.get_prefname(os.path.basename(dd)), # os.path.basename(dd)) # for dd in glob.glob(os.path.join(PARFILE_DIR, '*')) # if os.path.isdir(dd)]) #if psrname in psrdirs: # legacydir = os.path.join('/homes/plazarus/research/epta-legacy/', # psrdirs[psrname]) #else: # legacydir = None # Copy EPTA legacy TOAs #if legacydir and not os.path.exists("epta-legacy"): # os.mkdir("epta-legacy") # shutil.copytree(os.path.join(legacydir, "tims"), "epta-legacy/tims") # shutil.copy(os.path.join(legacydir, # "%s_t2noise.model" % psrdirs[psrname]), # "epta-legacy") # Find parfile if args.parfile is not None: if not os.path.exists(args.parfile): raise errors.InputError("Parfile specified (%s) doesn't exist!" % args.parfile) inparfn = args.parfile else: # Create parfile #inparfn = os.path.join('/homes/plazarus/research/epta-legacy/', # psrdirs[psrname], "%s.par" % psrdirs[psrname]) inparfn = reduce_data.PARFILES[psrname] #intimfn = os.path.join('/homes/plazarus/research/epta-legacy/', # psrdirs[psrname], "%s_all.tim" % psrdirs[psrname]) outparfn = "%s.T2.par" % psrname with open(inparfn, 'r') as inff, open(outparfn, 'w') as outff: for line in inff: # Don't copy over JUMPs or EFACs to 'outff' if not line.startswith("JUMP") and \ not 'EFAC' in line: outff.write(line) outff.write("\n".join(EXTRA_PARFILE_LINES)) template_dir = os.path.join(BASE_TEMPLATE_DIR, psrname) for stage in STAGES: if stage == "current": continue for rcvr in RCVRS: template_name = "%s_%s_%s.std" % (psrname, rcvr, stage) # First, check if templates exists if not os.path.isfile(os.path.join(template_dir, template_name)): # Make template utils.print_info("No template (%s) found!" % template_name, 1) try: os.makedirs(template_dir) except: pass try: print psrname, stage, rcvr stdfn = make_template.make_template( template_dir, psrname, stage, rcvr) utils.print_info("Made template: %s" % stdfn, 1) except errors.TemplateGenerationError: pass timfns = [] for stage in STAGES: # List files to reduce rows = list_files.get_files([psrname], stage) print len(rows) fns = {} # Initialize list of file names for each receiver for rcvr in RCVRS: fns[rcvr] = [] for row in rows: if row['stage'] not in ('cleaned', 'calibrated'): continue fn = os.path.join(row['filepath'], row['filename']) fns[row['rcvr']].append(fn) stagetimfn = "%s_%s.tim" % (psrname, stage) print "Opening %s" % stagetimfn stagetimff = open(stagetimfn, 'w') # Create file listings and generate TOAs for rcvr in RCVRS: print rcvr, len(fns[rcvr]) if not fns[rcvr]: # No files continue # Check for existing scrunched files toscrunch = [] scrunchedfns = [] scrunchdir = os.path.join("scrunched", rcvr) for fn in fns[rcvr]: scrunchfn = os.path.join(scrunchdir, os.path.basename(fn) + ext) scrunchedfns.append(scrunchfn) if not os.path.exists(scrunchfn): toscrunch.append(fn) # Scrunch files try: os.makedirs(scrunchdir) except: pass print "Working on %s %s" % (rcvr, stage) for fn in utils.show_progress(toscrunch, width=50): # Create a copy of the file with the 'eff_psrix' site cmd = ['psredit', '-c', 'site=eff_psrix', '-O', scrunchdir, fn] cmd.extend(['-e', fn.split('.')[-1] + ext]) utils.execute(cmd) arfn = os.path.join(scrunchdir, os.path.basename(fn + ext)) parfn = utils.get_norm_parfile(arfn) # Re-install ephemeris cmd = ['pam', '-Tp', '-E', parfn, '-m', arfn] + scrunchargs utils.execute(cmd) toas = [] mjds = [] for row in rows: if row['rcvr'] != rcvr: continue if row['stage'] not in ('cleaned', 'calibrated'): continue template_name = "%s_%s_%s.std" % (psrname, rcvr, row['stage']) template = os.path.join(template_dir, template_name) # Generate TOAs fn = os.path.join(scrunchdir, row['filename']) + ext print fn stdout, stderr = utils.execute([ "pat", "-T", "-A", "FDM", "-f", "tempo2", "-C", "rcvr chan", "-d", "-s", template, fn ]) # Parse TOAs toalines = stdout.split('\n') for line in toalines: toainfo = readers.tempo2_reader(line) if toainfo is not None: # Formatter expects 'file' field to be called 'rawfile' toainfo['rawfile'] = toainfo['file'] toainfo['telescope_code'] = toainfo['telescope'] toainfo['type'] = stage toainfo['rcvr'] = rcvr toainfo['file_id'] = row['file_id'] toainfo['obs_id'] = row['obs_id'] toainfo['shortstage'] = row['stage'][:2].upper() if row['stage'] == 'cleaned': toainfo['grp'] = "%s_clean" % rcvr else: toainfo['grp'] = "%s_cal" % rcvr toainfo['chan'] = toainfo['extras']['chan'] toas.append(toainfo) mjds.append(toainfo['imjd']) # Sort TOAs utils.sort_by_keys(toas, ['fmjd', 'imjd']) # Format timfile sysflag = 'EFF.AS.%(rcvr)s.%(shortstage)s' timlines = formatters.tempo2_formatter(toas, flags=[ ('rcvr', '%(rcvr)s'), ('type', '%(type)s'), ('grp', '%(grp)s'), ('sys', sysflag), ('obsid', '%(obs_id)d'), ('fileid', '%(file_id)d'), ('chan', '%(chan)s') ]) mjds.sort() #offsetmjds = sorted(TIME_OFFSETS.keys()) #inds = np.searchsorted(mjds, offsetmjds)+1 # Insert extra lines from back of list #for ind, key in reversed(zip(inds, offsetmjds)): # timlines[ind:ind] = ["\n"+TIME_OFFSETS[key]+"\n"] # Write out timfile timfn = "%s_%s_%s.tim" % (psrname, rcvr, stage) with open(timfn, 'w') as ff: for line in timlines: ff.write(line + "\n") utils.print_info("Wrote out timfile: %s" % timfn) timfns.append(timfn) stagetimff.write("INCLUDE %s\n" % timfn) stagetimff.close() #outtimfn = os.path.join("epta-legacy", os.path.basename(intimfn)) #with open(intimfn, 'r') as inff, open(outtimfn, 'w') as outff: # for line in inff: # outff.write(line) # for rcvr in RCVRS: # timfn = "%s_%s_cleaned.tim" % (psrname, rcvr) # if os.path.exists(timfn): # outff.write("INCLUDE ../%s\n" % timfn) # Count TOAs #toas = load_toa.parse_timfile(outtimfn, determine_obssystem=False) systems = {} #for toa in toas: # if toa['is_bad']: # continue # if not 'sys' in toa['extras']: # print toa # else: # nn = systems.get(toa['extras']['sys'], 0) # systems[toa['extras']['sys']] = nn+1 outparfn = "%s.T2.par" % psrname #outparfn2 = os.path.join("epta-legacy", os.path.basename(inparfn)) with open(inparfn, 'r') as inff, open(outparfn, 'w') as outff: #, \ #open(outparfn2, 'w') as outff2: for line in inff: # Don't copy over JUMPs or EFACs to 'outff' # Copy JUMPs and EFACs to 'outff2' and fit #if line.startswith("JUMP"): # if "-sys" in line: # obssys = line.split()[2] # if systems.get(obssys, 0): # # Observing system has TOAs # # Replace all system jumps by 0 and set the fit flag # outff2.write(" ".join(line.split()[:3])+" 0 1\n") # else: # outff2.write(line) #elif line.startswith("T2EFAC"): # outff2.write(line) #elif line.startswith("NITS"): # pass #else: outff.write(line) # Remove fit-flags for 'outff2' #outff2.write(" ".join(line.split()[:2])+'\n') outff.write("\n".join(EXTRA_PARFILE_LINES)) #outff2.write("\n".join(["JUMP -sys EFF.AS.%s.CL 0 1" % rcvr for rcvr in RCVRS])) #outff2.write("\nNITS 3\n") # Create a master timfile master_timfn = "%s_all.tim" % psrname with open(master_timfn, 'w') as ff: for timfn in timfns: ff.write("INCLUDE %s\n" % timfn) utils.print_info("Wrote out master timfile: %s" % master_timfn)
def combine_subints(subdirs, subints, parfn=None, outdir=None): """Combine sub-ints from various freq sub-band directories. The input lists are as created by 'group_subband_dirs' or read-in by 'read_listing'. Inputs: subdirs: List of sub-band directories containing sub-ints to combine subints: List of subint files to be combined. (NOTE: These are the file name only (i.e. no path) Each file listed should appear in each of the subdirs.) parfn: New ephemeris to install when combining subints. (Default: Use ephemeris in archive file's header) outdir: Directory to output combined file. (Default: Current working directory) Output: outfn: The name of the combined file. """ if outdir is None: outdir = os.getcwd() subints = sorted(subints) tmpdir = tempfile.mkdtemp(suffix="_combine", dir=config.tmp_directory) devnull = open(os.devnull) try: cmbsubints = [] # Try to normalise the archive's parfile try: if parfn is None: arfn = os.path.join(subdirs[0], subints[0]) normparfn = utils.get_norm_parfile(arfn) else: normparfn = utils.normalise_parfile(parfn) except errors.InputError: # No parfile present parargs = [] else: parargs = ['-E', normparfn] utils.print_info("Adding freq sub-bands for each sub-int...", 2) for ii, subint in enumerate(utils.show_progress(subints, width=50)): to_combine = [os.path.join(path, subint) for path in subdirs] outfn = os.path.join(tmpdir, "combined_%s" % subint) cmbsubints.append(outfn) utils.execute(['psradd', '-q', '-R', '-o', outfn] + parargs + to_combine, stderr=devnull) arf = utils.ArchiveFile( os.path.join(tmpdir, "combined_%s" % subints[0])) outfn = os.path.join( outdir, "%s_%s_%s_%05d_%dsubints.cmb" % (arf['name'], arf['band'], arf['yyyymmdd'], arf['secs'], len(subints))) utils.print_info("Combining %d sub-ints..." % len(cmbsubints), 1) utils.execute(['psradd', '-q', '-o', outfn] + cmbsubints, stderr=devnull) except: raise # Re-raise the exception finally: if debug.is_on('reduce'): warnings.warn("Not cleaning up temporary directory (%s)" % tmpdir, \ errors.CoastGuardWarning) else: utils.print_info("Removing temporary directory (%s)" % tmpdir, 2) shutil.rmtree(tmpdir) return outfn
def update_caldb(db, sourcename, force=False): """Check for new calibrator scans. If found update the calibrator database. Inputs: db: A Database object. sourcename: The name of the source to match. (NOTE: '_R' will be removed from the sourcename, if present) force: Forcefully update the caldb Outputs: caldb: The path to the updated caldb. """ name = utils.get_prefname(sourcename) if name.endswith('_R'): name = name[:-2] # Get the caldb caldb = get_caldb(db, name) if caldb is None: lastupdated = datetime.datetime.min outdir = os.path.join(config.output_location, 'caldbs') try: os.makedirs(outdir) except OSError: # Directory already exists pass outfn = '%s.caldb.txt' % name.upper() outpath = os.path.join(outdir, outfn) insert_new = True values = {'sourcename': name, 'caldbpath': outdir, 'caldbname': outfn} else: lastupdated = caldb['last_modified'] outpath = os.path.join(caldb['caldbpath'], caldb['caldbname']) insert_new = False values = {} with db.transaction() as conn: if not insert_new: # Mark update of caldb as in-progress update = db.caldbs.update().\ values(status='updating', last_modified=datetime.datetime.now()).\ where(db.caldbs.c.caldb_id == caldb['caldb_id']) conn.execute(update) select = db.select([db.files], from_obj=[db.files.\ outerjoin(db.obs, onclause=db.files.c.obs_id == db.obs.c.obs_id)]).\ where((db.files.c.status.in_(['new', 'done'])) & (db.files.c.stage == 'calibrated') & (db.obs.c.obstype == 'cal') & (db.obs.c.sourcename == ('%s_R' % name))) results = conn.execute(select) rows = results.fetchall() results.close() numnew = 0 for row in rows: if row['added'] > lastupdated: numnew += 1 utils.print_info("Found %d suitable calibrators for %s. " "%d are new." % (len(rows), name, numnew), 2) values['numentries'] = len(rows) try: if numnew or force: # Create an updated version of the calibrator database basecaldir = os.path.join(config.output_location, name.upper()+"_R") utils.execute(['pac', '-w', '-u', '.pcal.T', '-k', outpath], dir=basecaldir) except: #raise values['status'] = 'failed' if insert_new: action = db.caldbs.insert() else: action = db.caldbs.update().\ values(note='%d new entries added' % numnew, last_modifed=datetime.datetime.now()).\ where(db.caldbs.c.caldb_id == caldb['caldb_id']) conn.execute(action, values) else: if insert_new: action = db.caldbs.insert() else: action = db.caldbs.update().\ values(status='ready', note='%d new entries added' % numnew, last_modified=datetime.datetime.now()).\ where(db.caldbs.c.caldb_id == caldb['caldb_id']) conn.execute(action, values) return outpath