def add_timfile_entry(toas, cmdline, comments, conflict_handler, existdb=None): """Insert a timfile entry in the DB, and associate TOAs with it. Inputs: toas: A list of row objects each representing a TOA. cmdline: the command line used when running the program. comments: User comments describing the timfile. conflict_handler: A handler function to use. existdb: A (optional) existing database connection object. (Default: Establish a db connection) Output: timfile_id: The resulting ID of the timfile entry. """ # Check for / handle conflicts toas = conflict_handler(toas) if not toas: raise errors.ToasterError("No TOAs match criteria provided!") # Connect to DB, if not using an already-established connection db = existdb or database.Database() db.connect() # Insert timfile entry ins = db.timfiles.insert() values = {'user_id': cache.get_userid(), 'version_id': version.get_version_id(db), 'comments': comments, 'pulsar_id': toas[0]['pulsar_id'], 'input_args': cmdline} result = db.execute(ins, values) timfile_id = result.inserted_primary_key[0] result.close() # Associate the TOAs ins = db.toa_tim.insert() values = [] for toa in toas: values.append({'timfile_id': timfile_id, 'toa_id': toa['toa_id']}) db.execute(ins, values) if not existdb: db.close() return timfile_id
def pipeline_core(manip, rawfile_id, parfile_id, template_id, existdb=None): """Run a prepared manipulator function on the raw file with ID 'rawfile_id'. Then generate TOAs and load them into the DB. Inputs: manip: A manipulator instance. rawfile_id: The ID number of the raw data file to generate TOAs from. parfile_id: The ID number of the parfile to install into the raw file. If this is None, then no new parfile will be installed. template_id: The ID number of the template to use. existdb: An existing database connection object. (Default: establish a new DB connection) Outputs: None """ # Initialise these so the 'finally' clause doesn't throw an exception of # it's own if an error is caught before these filenames are determined manipfn = '' adjustfn = '' #Start pipeline print "###################################################" print "Starting to toast data" print "Start time: %s" % utils.give_utc_now() print "###################################################" db = existdb or database.Database() db.connect() try: trans = db.begin() # Open a transaction # Get version ID version_id = version.get_version_id(db) # Get raw data from rawfile_id and verify MD5SUM rawfile = rawfiles_general.get_rawfile_from_id(rawfile_id, db, verify_md5=True) # Manipulate the raw file notify.print_info("Manipulating file", 1) # Create a temporary file for the adjusted results tmpfile, adjustfn = tempfile.mkstemp(prefix='toaster_tmp', suffix='_newephem.ar', dir=config.cfg.base_tmp_dir) os.close(tmpfile) shutil.copy(rawfile, adjustfn) if parfile_id is not None: # Re-install ephemeris # Get ephemeris from parfile_id and verify MD5SUM parfile = parfiles_general.get_parfile_from_id(parfile_id, db, verify_md5=True) cmd = ["pam", "-m", "-E", parfile, "--update_dm", adjustfn] utils.execute(cmd) # Create a temporary file for the manipulated results tmpfile, manipfn = tempfile.mkstemp(prefix='toaster_tmp', suffix='_manip.ar', dir=config.cfg.base_tmp_dir) os.close(tmpfile) # Run the manipulator manip.run([adjustfn], manipfn, tmpdir=config.cfg.base_tmp_dir) # Get template from template_id and verify MD5SUM template = templates_general.get_template_from_id(template_id, db, verify_md5=True) # Create a temporary file for the toa diagnostic plots tmpfile, toadiagfn = tempfile.mkstemp(prefix='toaster_tmp', suffix='_TOAdiag.png', dir=config.cfg.base_tmp_dir) os.close(tmpfile) # Generate TOAs with pat notify.print_info("Computing TOAs", 0) cmd = ["pat", "-f", "tempo2", "-A", config.cfg.toa_fitting_method, "-s", template, "-C", "gof length bw nbin nchan nsubint", "-t", "-K", "%s/PNG" % toadiagfn, manipfn] patout, paterr = utils.execute(cmd) # Check version ID is still the same. Just in case. new_version_id = version.get_version_id(db) if version_id != new_version_id: raise errors.ToasterError("Weird... Version ID at the start " "of processing (%s) is different " "from at the end (%d)!" % (version_id, new_version_id)) # Read some header values from the manipulated archive hdr = datafile.get_header_vals(manipfn, ['nchan', 'nsub', 'name', 'intmjd', 'fracmjd']) hdr['secs'] = int(hdr['fracmjd']*24*3600+0.5) # Add 0.5 so result is # rounded to nearest int # Fill pipeline table cmdline = " ".join(sys.argv) process_id = fill_process_table(version_id, rawfile_id, parfile_id, template_id, manip, hdr['nchan'], hdr['nsub'], db) # Parse pat output toainfo = toas_general.parse_pat_output(patout) rawfile_info = rawfiles_general.get_rawfile_info(rawfile_id) # Insert TOAs into DB for ti in toainfo: ti['process_id'] = process_id ti['template_id'] = template_id ti['rawfile_id'] = rawfile_id ti['pulsar_id'] = rawfile_info['pulsar_id'] ti['obssystem_id'] = rawfile_info['obssystem_id'] toa_ids = load_toa.load_toas(toainfo, db) # Create processing diagnostics notify.print_info("Generating processing diagnostics", 1) diagdir = make_proc_diagnostics_dir(manipfn, process_id) suffix = "_procid%d.%s" % (process_id, manip.name) diags = [] for diagname in config.cfg.default_rawfile_diagnostics: diagcls = diagnostics.get_diagnostic_class(diagname) try: diags.append(diagcls(manipfn)) except errors.DiagnosticNotApplicable, e: notify.print_info("Diagnostic isn't applicable: %s. " "Skipping..." % str(e), 1) if diags: # Load processing diagnostics diagnose_processing.insert_processing_diagnostics(process_id, diags, diagdir, suffix, existdb=db) # Copy TOA diagnostic plots and register them into DB basefn = "%(name)s_%(intmjd)05d_%(secs)05d" % hdr values = [] for ii, toa_id in enumerate(toa_ids): outfn = basefn+"_procid%d.TOA%d.png" % (process_id, ii+1) if ii == 0: fn = toadiagfn else: fn = "%s_%d" % (toadiagfn, ii+1) shutil.move(fn, os.path.join(diagdir, outfn)) ins = db.toa_diagnostic_plots.insert() values.append({'toa_id': toa_id, 'filename': outfn, 'filepath': diagdir, 'plot_type': 'Prof-Temp Resids'}) result = db.execute(ins, values) result.close() notify.print_info("Inserted %d TOA diagnostic plots." % len(toa_ids), 2)