Example #1
0
def get_template_from_id(template_id, existdb=None, verify_md5=True):
    """Return the path to the raw file that has the given ID number.
        Optionally double check the file's MD5 sum, to make sure
        nothing strange has happened.

        Inputs:
            template_id: The ID number of the raw file to get
                a path for.
            existdb: A (optional) existing database connection object.
                (Default: Establish a db connection)
            verify_md5: If True, double check the file's MD5 sum.
                (Default: Perform MD5 check.)

        Output:
            fn: The full file path.
    """
    notify.print_info("Looking-up raw file with ID=%d" % template_id, 2)

    # Use the exisitng DB connection, or open a new one if None was provided
    db = existdb or database.Database()
    db.connect()
    
    select = db.select([db.templates.c.filename,
                        db.templates.c.filepath,
                        db.templates.c.md5sum]).\
                where(db.templates.c.template_id == template_id)
    result = db.execute(select)
    rows = result.fetchall()
    result.close()
    if not existdb:
        # Close the DB connection we opened
        db.close()

    if len(rows) == 1:
        filename = rows[0]['filename']
        filepath = rows[0]['filepath']
        md5sum_db = rows[0]['md5sum']
    else:
        raise errors.InconsistentDatabaseError("Bad number of files (%d) "
                                               "with template_id=%d" %
                                               (len(rows), template_id))
        
    fullpath = os.path.join(filepath, filename)
    # Make sure the file exists
    datafile.verify_file_path(fullpath)
    if verify_md5:
        notify.print_info("Confirming MD5 sum of %s matches what is "
                          "stored in DB (%s)" % (fullpath, md5sum_db), 2)
                    
        md5sum_file = datafile.get_md5sum(fullpath)
        if md5sum_db != md5sum_file:
            raise errors.FileError("md5sum check of %s failed! MD5 from "
                                   "DB (%s) != MD5 from file (%s)" %
                                   (fullpath, md5sum_db, md5sum_file))
    return fullpath
Example #2
0
def populate_parfiles_table(db, fn, params):
    # md5sum helper function in utils 
    md5 = datafile.get_md5sum(fn)
    path, fn = os.path.split(os.path.abspath(fn))
   
    db.begin()  # Begin a transaction
    # Does this file exist already?
    select = db.select([db.parfiles.c.parfile_id, db.parfiles.c.pulsar_id],
                        db.parfiles.c.md5sum == md5)
    result = db.execute(select)
    rows = result.fetchall()
    result.close()
    if len(rows) > 1:
        db.rollback()
        raise errors.InconsistentDatabaseError("There are %d parfiles "
                                               "with MD5 (%s) in the "
                                               "database already" %
                                               (len(rows), md5))
    elif len(rows) == 1:
        parfile_id, psr_id = rows[0]
        if psr_id == params['pulsar_id']:
            warnings.warn("A parfile with this MD5 (%s) already exists "
                          "in the DB for this pulsar (ID: %d). "
                          "The file will not be re-registed into the DB. "
                          "Doing nothing..." % (md5, psr_id),
                          errors.ToasterWarning)
        else:
            db.rollback()
            raise errors.InconsistentDatabaseError("A parfile with this "
                                                   "MD5 (%s) already exists "
                                                   "in the DB, but for "
                                                   "a different pulsar "
                                                   "(ID: %d)!" % (md5, psr_id))
    else:
        # Based on its MD5, this parfile doesn't already 
        # exist in the DB. Insert it.

        # Insert the parfile
        ins = db.parfiles.insert()
        values = {'md5sum': md5,
                  'filename': fn,
                  'filepath': path}

        values.update(params)
        result = db.execute(ins, values)
        parfile_id = result.inserted_primary_key[0]
        result.close()
    
    db.commit()
    return parfile_id 
Example #3
0
def get_template_id(template, existdb=None):
    """Given a template file path find its template_id number.
        
        Inputs:
            template: the path to a template file.
            existdb: A (optional) existing database connection object.
                (Default: Establish a db connection)

        Output:
            template_id: the corresponding template_id value.
    """
    # Use the exisitng DB connection, or open a new one if None was provided
    db = existdb or database.Database()
    db.connect()
    
    notify.print_info("Getting template ID for %s using "
                      "filename and md5sum" % template, 2)
    path, fn = os.path.split(os.path.abspath(template))
    md5sum = datafile.get_md5sum(template)
    select = db.select([db.templates.c.template_id,
                        db.templates.c.filename,
                        db.templates.c.md5sum]).\
                where((db.template.c.md5sum == md5sum) |
                      (db.templates.c.filename == fn))
    result = db.execute(select)
    rows = result.fetchall()
    result.close()
    if not existdb:
        db.close()
    if len(rows) == 1:
        row = rows[0]
        if row['md5sum'] == md5sum and row['filename'] == fn:
            return row['template_id']
        elif row['md5sum'] == md5sum:
            raise errors.FileError("A template (template_id=%d) with "
                                   "this md5sum, but a different filename "
                                   "exists in the DB." % row['template_id'])
        elif row['filename'] == fn:
            raise errors.FileError("A template (template_id=%d) with "
                                   "this filename, but a different md5sum "
                                   "exists in the DB." % row['template_id'])
        else:
            raise errors.InconsistentDatabaseError("A template "
                                                   "(template_id=%d) "
                                                   "matches our query, but "
                                                   "neither its md5sum (%s), "
                                                   "nor its filename (%s) "
                                                   "appears to match! "
                                                   "This should never "
                                                   "happen!" %
                                                   (row['template_id'],
                                                    row['md5sum'], row['fn']))
    elif len(rows) == 0:
        raise errors.ToasterError("Input template (%s) does not appear "
                                  "to be registered in the DB! "
                                  "Use 'load_template.py' to load "
                                  "it into the DB." % template)
    else:
        raise errors.InconsistentDatabaseError("Multiple (%s) templates "
                                               "match the given file name "
                                               "or md5sum!" % len(rows))
Example #4
0
def populate_templates_table(db, fn, params, comments):
    if comments is None:
        raise errors.BadInputError("A comment is required for every "
                                   "template!")
    # md5sum helper function in utils 
    md5 = datafile.get_md5sum(fn)
    path, fn = os.path.split(os.path.abspath(fn))
    
    trans = db.begin()
    # Does this file exist already?
    select = db.select([db.templates.c.template_id,
                        db.templates.c.pulsar_id]).\
                where(db.templates.c.md5sum == md5)
    results = db.execute(select)
    rows = results.fetchall()
    results.close()
    if len(rows) > 1:
        db.rollback()
        raise errors.InconsistentDatabaseError("There are %d templates "
                                               "with MD5 (%s) in the "
                                               "database already" %
                                               (len(rows), md5))
    elif len(rows) == 1:
        psr_id = rows[0]['pulsar_id']
        template_id = rows[0]['template_id']
        if psr_id == params['pulsar_id']:
            db.commit()
            warnings.warn("A template with this MD5 (%s) already exists "
                          "in the DB for this pulsar (ID: %d). "
                          "The file will not be re-registed into the DB. "
                          "Doing nothing..." % (md5, psr_id),
                          errors.ToasterWarning)
            return template_id
        else:
            db.rollback()
            raise errors.InconsistentDatabaseError("A template with this "
                                                   "MD5 (%s) already exists "
                                                   "in the DB, but for "
                                                   "a different pulsar "
                                                   "(ID: %d)!" %
                                                   (md5, psr_id))
    else:
        # Based on its MD5, this template doesn't already 
        # exist in the DB.

        # Check to see if this pulsar/observing system combination
        # Already has a template
        select = db.select([db.templates.c.template_id]).\
                    where((db.templates.c.pulsar_id == params['pulsar_id']) &
                          (db.templates.c.obssystem_id == params['obssystem_id']))
        results = db.execute(select)
        rows = results.fetchall()
        results.close()
        if len(rows):
            warnings.warn("This pulsar_id (%d), obssystem_id (%d) "
                          "combination already has %d templates in the DB. "
                          "Be sure to correctly set the master template." %
                          (params['pulsar_id'], params['obssystem_id'],
                           len(rows)))

        # Insert the template
        ins = db.templates.insert()
        values = {'md5sum': md5,
                  'filename': fn,
                  'filepath': path,
                  'user_id': params['user_id'],
                  'pulsar_id': params['pulsar_id'],
                  'obssystem_id': params['obssystem_id'],
                  'nbin': params['nbin'],
                  'comments': comments}
        result = db.execute(ins, values)
        template_id = result.inserted_primary_key[0]
        result.close()
    db.commit()
    return template_id 
Example #5
0
def populate_rawfiles_table(db, archivefn, params):
    # md5sum helper function in utils 
    md5 = datafile.get_md5sum(archivefn)
    path, fn = os.path.split(os.path.abspath(archivefn))
    size = os.path.getsize(archivefn)  # File size in bytes

    trans = db.begin()
    # Does this file exist already?
    select = db.select([db.rawfiles.c.rawfile_id,
                        db.rawfiles.c.pulsar_id]).\
                where(db.rawfiles.c.md5sum == md5)
    result = db.execute(select)
    rows = result.fetchall()
    result.close()
    if len(rows) > 1:
        trans.rollback()
        raise errors.InconsistentDatabaseError("There are %d rawfiles "
                                               "with MD5 (%s) in the "
                                               "database already" %
                                               (len(rows), md5))
    elif len(rows) == 1:
        rawfile_id = rows[0]['rawfile_id']
        psr_id = rows[0]['pulsar_id']
        if psr_id == params['pulsar_id']:
            warnings.warn("A rawfile with this MD5 (%s) already exists "
                          "in the DB for this pulsar (ID: %d). "
                          "The file will not be re-registed into the DB. "
                          "Doing nothing..." % (md5, psr_id),
                          errors.ToasterWarning)
            trans.commit()
            return rawfile_id
        else:
            trans.rollback()
            raise errors.InconsistentDatabaseError("A rawfile with this "
                                                   "MD5 (%s) already exists "
                                                   "in the DB, but for "
                                                   "a different pulsar "
                                                   "(ID: %d)!" %
                                                   (md5, psr_id))
    else:
        notify.print_info("Inserting rawfile (%s) into DB." % fn, 3)
        # Based on its MD5, this rawfile doesn't already 
        # exist in the DB. Insert it.

        # Insert the file
        ins = db.rawfiles.insert()
        values = {'md5sum': md5,
                  'filename': fn,
                  'filepath': path,
                  'filesize': size,
                  'coord': '%s,%s' % (params['ra'], params['dec'])}
        values.update(params)
        result = db.execute(ins, values)
        rawfile_id = result.inserted_primary_key[0]
        result.close()

        # Create rawfile diagnostics
        diags = []
        for diagname in config.cfg.default_rawfile_diagnostics:
            diagcls = diagnostics.get_diagnostic_class(diagname)
            try:
                diags.append(diagcls(archivefn))
            except errors.DiagnosticNotApplicable, e:
                notify.print_info("Diagnostic isn't applicable: %s. "
                                  "Skipping..." % str(e), 1)
        if diags:
            # Load processing diagnostics
            diagnose_rawfile.insert_rawfile_diagnostics(rawfile_id, diags,
                                                        existdb=db)