def __plot_nofreq(self, tmpfn, params): info = self.__get_info(params) cmd = [ "psrplot", "-O", "-j", "D", "-c", "above:c=,x:range=0:2", self.fn, "-D", "%s/PNG" % tmpfn, "-p", "flux", "-c", ":0:x:view=0.075:0.95," "y:view=0.5:0.7," "subint=I," "chan=I," "pol=I," "x:opt=BCTS," "x:lab=," "below:l=," "%s" % info, "-p", "time", "-c", ":1:x:view=0.075:0.95," "y:view=0.15:0.5," "chan=I," "pol=I," "cmap:map=plasma", ] utils.execute(cmd)
def show_templates(templates): if len(templates): print "--"*25 for tdict in templates: print colour.cstring("Template ID:", underline=True, bold=True) + \ colour.cstring(" %d" % tdict['template_id'], bold=True) fn = os.path.join(tdict['filepath'], tdict['filename']) print "\nTemplate: %s" % fn print "Pulsar name: %s" % tdict['pulsar_name'] print "Master template? %s" % \ (((tdict['mtempid'] is not None) and "Yes") or "No") print "Number of phase bins: %d" % tdict['nbin'] print "Uploaded by: %s (%s)" % (tdict['real_name'], tdict['email_address']) print "Uploader's comments: %s" % tdict['comments'] print "Date and time template was added: %s" % \ tdict['add_time'].isoformat(' ') # Show extra information if verbosity is >= 1 lines = ["Observing System ID: %d" % tdict['obssystem_id'], "Observing System Name: %s" % tdict['obssys_name'], "Telescope: %s" % tdict['telescope_name'], "Frontend: %s" % tdict['frontend'], "Backend: %s" % tdict['backend'], "Clock: %s" % tdict['clock']] notify.print_info("\n".join(lines), 1) try: # Show the template if verbosity is >= 2 cmd = ["psrtxt", fn] psrtxtout, stderr = utils.execute(cmd) gnuplotcode = """set term dumb set format y "" set nokey set border 1 set tics out set xtics nomirror set ytics 0,1,0 set xlabel "Phase Bin" set xrange [0:%d] plot "-" using 3:4 w l %s end """ % (tdict.nbin-1, psrtxtout) plot, stderr = utils.execute(["gnuplot"], stderr=open(os.devnull), stdinstr=gnuplotcode) notify.print_info(plot, 2) except errors.SystemCallError: # gnuplot is probably not installed pass print "--"*25 else: raise errors.ToasterError("No templates match parameters provided!")
def parse_psrfits_header(fn, hdritems): """Get a set of header params from the given file. Returns a dictionary. Inputs: fn: The name of the file to get params for. hdritems: List of parameter names to fetch. Output: params: A dictionary. The keys are values requested from 'psredit' the values are the values reported by 'psredit'. """ hdrstr = ",".join(hdritems) if '=' in hdrstr: raise ValueError("'hdritems' passed to 'parse_psrfits_header' " "should not perform and assignments!") cmd = ["psredit", "-q", "-Q", "-c", hdrstr, fn] outstr, errstr = utils.execute(cmd) outvals = outstr.split() if errstr: raise errors.SystemCallError("The command: %s\nprinted to stderr:\n%s" % (cmd, errstr)) elif len(outvals) != len(hdritems): raise errors.SystemCallError("The command: %s\nreturn the wrong " "number of values. (Was expecting %d, got %d.)" % (cmd, len(hdritems), len(outvals))) params = {} for key, val in zip(hdritems, outstr.split()): params[key] = val return params
def _compute(self): notify.print_info("Calling psrstat to get weights for %s" % self.fn, 3) cmd = ["psrstat", self.fn, "-c", "int:wt", "-Qq"] outstr, errstr = utils.execute(cmd) wtstrs = outstr.strip().split(',') weights = np.array([float(wt) for wt in wtstrs]) maskpcnt = 100.0*np.sum(weights > 0)/weights.size return maskpcnt
def __plot_profonly(self, tmpfn, params): info = self.__get_info(params) cmd = [ "psrplot", "-O", "-j", "D", "-c", "above:c=,x:range=0:2", self.fn, "-D", "%s/PNG" % tmpfn, "-p", "flux", "-c", ":0:x:view=0.075:0.95," "y:view=0.15:0.7," "subint=I," "chan=I," "pol=I," "below:l=," "%s" % info, ] utils.execute(cmd)
def get_githash(repodir): """Get the pipeline's git hash. Inputs: repodir: The location of the git repository. Output: githash: The githash """ if is_gitrepo_dirty(repodir): warnings.warn("Git repository has uncommitted changes!", \ errors.ToasterWarning) cmd = ["git", "rev-parse", "HEAD"] stdout, stderr = utils.execute(cmd, execdir=repodir) githash = stdout.strip() return githash
def get_header_vals(fn, hdritems): """Get a set of header params from the given file. Returns a dictionary. Inputs: fn: The name of the file to get params for. hdritems: List of parameters (recognized by vap) to fetch. Output: params: A dictionary. The keys are values requested from 'vap' the values are the values reported by 'vap'. """ if not len(hdritems): raise ValueError("No 'hdritems' requested to get from file header!") hdrstr = ",".join(hdritems) if '=' in hdrstr: raise ValueError("'hdritems' passed to 'get_header_vals' " "should not perform and assignments!") cmd = ["vap", "-n", "-c", hdrstr, fn] outstr, errstr = utils.execute(cmd) outvals = outstr.split()[(0 - len(hdritems)):] # First value is filename (we don't need it) if errstr: raise errors.SystemCallError("The command: %s\nprinted to stderr:\n%s" % (cmd, errstr)) elif len(outvals) != len(hdritems): raise errors.SystemCallError("The command: %s\nreturned the wrong " "number of values. (Was expecting %d, got %d.)" % (cmd, len(hdritems), len(outvals))) params = HeaderParams(fn) for key, val in zip(hdritems, outvals): if val == "INVALID": raise errors.SystemCallError("The vap header key '%s' " "is invalid!" % key) elif val == "*" or val == "UNDEF": warnings.warn("The vap header key '%s' is not " "defined in this file (%s)" % (key, fn), errors.ToasterWarning) params[key] = None else: # Get param's type to cast value caster = header_param_types.get(key, str) params[key] = caster(val) return params
def is_gitrepo_dirty(repodir): """Return True if the git repository has local changes. Inputs: repodir: The location of the git repository. Output: is_dirty: True if git repository has local changes. False otherwise. """ notify.print_info("Checking if Git repo at '%s' is dirty..." % repodir, 2) try: cmd = ["git", "diff", "--quiet"] stdout, stderr = utils.execute(cmd, execdir=repodir) except errors.SystemCallError: # Exit code is non-zero return True else: # Success error code (i.e. no differences) return False
def is_gitrepo(repodir): """Return True if the given dir is a git repository. Input: repodir: The location of the git repository. Output: is_git: True if directory is part of a git repository. False otherwise. """ notify.print_info("Checking if directory '%s' contains a Git repo..." % repodir, 2) try: cmd = ["git", "rev-parse"] stdout, stderr = utils.execute(cmd, execdir=repodir, \ stderr=open(os.devnull)) except errors.SystemCallError: # Exit code is non-zero return False else: # Success error code (i.e. dir is in a git repo) return True
def pipeline_core(manip, rawfile_id, parfile_id, template_id, existdb=None): """Run a prepared manipulator function on the raw file with ID 'rawfile_id'. Then generate TOAs and load them into the DB. Inputs: manip: A manipulator instance. rawfile_id: The ID number of the raw data file to generate TOAs from. parfile_id: The ID number of the parfile to install into the raw file. If this is None, then no new parfile will be installed. template_id: The ID number of the template to use. existdb: An existing database connection object. (Default: establish a new DB connection) Outputs: None """ # Initialise these so the 'finally' clause doesn't throw an exception of # it's own if an error is caught before these filenames are determined manipfn = '' adjustfn = '' #Start pipeline print "###################################################" print "Starting to toast data" print "Start time: %s" % utils.give_utc_now() print "###################################################" db = existdb or database.Database() db.connect() try: trans = db.begin() # Open a transaction # Get version ID version_id = version.get_version_id(db) # Get raw data from rawfile_id and verify MD5SUM rawfile = rawfiles_general.get_rawfile_from_id(rawfile_id, db, verify_md5=True) # Manipulate the raw file notify.print_info("Manipulating file", 1) # Create a temporary file for the adjusted results tmpfile, adjustfn = tempfile.mkstemp(prefix='toaster_tmp', suffix='_newephem.ar', dir=config.cfg.base_tmp_dir) os.close(tmpfile) shutil.copy(rawfile, adjustfn) if parfile_id is not None: # Re-install ephemeris # Get ephemeris from parfile_id and verify MD5SUM parfile = parfiles_general.get_parfile_from_id(parfile_id, db, verify_md5=True) cmd = ["pam", "-m", "-E", parfile, "--update_dm", adjustfn] utils.execute(cmd) # Create a temporary file for the manipulated results tmpfile, manipfn = tempfile.mkstemp(prefix='toaster_tmp', suffix='_manip.ar', dir=config.cfg.base_tmp_dir) os.close(tmpfile) # Run the manipulator manip.run([adjustfn], manipfn, tmpdir=config.cfg.base_tmp_dir) # Get template from template_id and verify MD5SUM template = templates_general.get_template_from_id(template_id, db, verify_md5=True) # Create a temporary file for the toa diagnostic plots tmpfile, toadiagfn = tempfile.mkstemp(prefix='toaster_tmp', suffix='_TOAdiag.png', dir=config.cfg.base_tmp_dir) os.close(tmpfile) # Generate TOAs with pat notify.print_info("Computing TOAs", 0) cmd = ["pat", "-f", "tempo2", "-A", config.cfg.toa_fitting_method, "-s", template, "-C", "gof length bw nbin nchan nsubint", "-t", "-K", "%s/PNG" % toadiagfn, manipfn] patout, paterr = utils.execute(cmd) # Check version ID is still the same. Just in case. new_version_id = version.get_version_id(db) if version_id != new_version_id: raise errors.ToasterError("Weird... Version ID at the start " "of processing (%s) is different " "from at the end (%d)!" % (version_id, new_version_id)) # Read some header values from the manipulated archive hdr = datafile.get_header_vals(manipfn, ['nchan', 'nsub', 'name', 'intmjd', 'fracmjd']) hdr['secs'] = int(hdr['fracmjd']*24*3600+0.5) # Add 0.5 so result is # rounded to nearest int # Fill pipeline table cmdline = " ".join(sys.argv) process_id = fill_process_table(version_id, rawfile_id, parfile_id, template_id, manip, hdr['nchan'], hdr['nsub'], db) # Parse pat output toainfo = toas_general.parse_pat_output(patout) rawfile_info = rawfiles_general.get_rawfile_info(rawfile_id) # Insert TOAs into DB for ti in toainfo: ti['process_id'] = process_id ti['template_id'] = template_id ti['rawfile_id'] = rawfile_id ti['pulsar_id'] = rawfile_info['pulsar_id'] ti['obssystem_id'] = rawfile_info['obssystem_id'] toa_ids = load_toa.load_toas(toainfo, db) # Create processing diagnostics notify.print_info("Generating processing diagnostics", 1) diagdir = make_proc_diagnostics_dir(manipfn, process_id) suffix = "_procid%d.%s" % (process_id, manip.name) diags = [] for diagname in config.cfg.default_rawfile_diagnostics: diagcls = diagnostics.get_diagnostic_class(diagname) try: diags.append(diagcls(manipfn)) except errors.DiagnosticNotApplicable, e: notify.print_info("Diagnostic isn't applicable: %s. " "Skipping..." % str(e), 1) if diags: # Load processing diagnostics diagnose_processing.insert_processing_diagnostics(process_id, diags, diagdir, suffix, existdb=db) # Copy TOA diagnostic plots and register them into DB basefn = "%(name)s_%(intmjd)05d_%(secs)05d" % hdr values = [] for ii, toa_id in enumerate(toa_ids): outfn = basefn+"_procid%d.TOA%d.png" % (process_id, ii+1) if ii == 0: fn = toadiagfn else: fn = "%s_%d" % (toadiagfn, ii+1) shutil.move(fn, os.path.join(diagdir, outfn)) ins = db.toa_diagnostic_plots.insert() values.append({'toa_id': toa_id, 'filename': outfn, 'filepath': diagdir, 'plot_type': 'Prof-Temp Resids'}) result = db.execute(ins, values) result.close() notify.print_info("Inserted %d TOA diagnostic plots." % len(toa_ids), 2)
def _compute(self): notify.print_info("Calling psrstat to get SNR for %s" % self.fn, 3) cmd = ["psrstat", "-Qq", "-j", "DTFp", "-c", "snr", self.fn] outstr, errstr = utils.execute(cmd) snr = float(outstr) return snr
def get_version_id(existdb=None): """Get the pipeline version number. If the version number isn't in the database, add it. Input: existdb: A (optional) existing database connection object. (Default: Establish a db connection) Output: version_id: The version ID for the current pipeline/psrchive combination. """ # Check to make sure the repositories are clean check_repos() # Get git hashes pipeline_githash = get_githash(os.path.dirname(__file__)) if is_gitrepo(config.cfg.psrchive_dir): psrchive_githash = get_githash(config.cfg.psrchive_dir) else: warnings.warn("PSRCHIVE directory (%s) is not a git repository! " \ "Falling back to 'psrchive --version' for version " \ "information." % config.cfg.psrchive_dir, \ errors.ToasterWarning) cmd = ["psrchive", "--version"] stdout, stderr = utils.execute(cmd) psrchive_githash = stdout.strip() # Use the exisitng DB connection, or open a new one if None was provided db = existdb or database.Database() db.connect() db.begin() # open a transaction # Check to see if this combination of versions is in the database select = db.select([db.versions.c.version_id]).\ where((db.versions.c.pipeline_githash==pipeline_githash) & \ (db.versions.c.psrchive_githash==psrchive_githash) & \ (db.versions.c.tempo2_cvsrevno=='Not available')) result = db.execute(select) rows = result.fetchall() result.close() if len(rows) > 1: db.rollback() if not existdb: # Close the DB connection we opened db.close() raise errors.DatabaseError("There are too many (%d) matching " \ "version IDs" % len(rows)) elif len(rows) == 1: version_id = rows[0].version_id else: # Insert the current versions ins = db.versions.insert() values = {'pipeline_githash':pipeline_githash, \ 'psrchive_githash':psrchive_githash, \ 'tempo2_cvsrevno':'Not available'} result = db.execute(ins, values) # Get the newly add version ID version_id = result.inserted_primary_key[0] result.close() db.commit() if not existdb: # Close the DB connection we opened db.close() return version_id