def main(): import argparse from idb.config import config parser = argparse.ArgumentParser( description='Generate a report of newly discovered recordsets.') parser.add_argument( '-a', '--age', help= 'Number of days ago from today to mark beginning of "recent" period.', default=31, type=int) parser.add_argument('-w', '--write', action='store_true', help='Write to a File instead of STDOUT.') args = parser.parse_args() db = PostgresDB() sql = """ select uuid, name, publisher_uuid, file_link, to_json(first_seen), to_json(pub_date), to_json(file_harvest_date) from recordsets where ingest=false and first_seen > now()-'%s days'::interval order by first_seen""" % args.age results_list = db.fetchall(sql) if args.write: fh = open("fresh-recordsets-report.tsv", "w") output_tsv(results_list, fh) fh.close() else: output_tsv(results_list, sys.stdout)
def upload_recordset_from_file(rsid, fname): """ Given a recordset uuid and a local dataset filename, upload the local dataset file as the "current" file for that uuid. Parameters ---------- rsid : uuid An iDigBio recordset uuid fname : string Filename (full path or current directory only) Returns ------- bool True if successful, False otherwise """ # convert rsid uuid to string here because of either: # psycopg2.ProgrammingError: can't adapt type 'UUID' # or # TypeError: 'UUID' object does not support indexing rsuuid = str(rsid) logger.info("Manual upload of '{0}' from file '{1}' requested.".format( rsuuid, fname)) # do some checks here try: f = open(fname) f.close() except: logger.error( "Cannot access file: '{0}'. Aborting upload.".format(fname)) raise db = PostgresDB() sql = ("""SELECT id FROM recordsets WHERE uuid=%s""", (rsuuid, )) idcount = db.execute(*sql) if idcount < 1: logger.error( "Cannot find uuid '{0}' in db. Aborting upload.".format(rsuuid)) db.rollback() return False # output the "before" state results = db.fetchall( """SELECT id,file_harvest_date,file_harvest_etag FROM recordsets WHERE uuid=%s""", (rsuuid, )) for each in results: logger.debug("{0}".format(each)) try: etag = upload_recordset(rsuuid, fname, db) assert etag sql = ("""UPDATE recordsets SET file_harvest_etag=%s, file_harvest_date=%s WHERE uuid=%s""", (etag, datetime.datetime.now(), rsuuid)) update_count = db.execute(*sql) db.commit() logger.info("UPDATED {0} rows.".format(update_count)) logger.info( "Finished manual upload of file '{0}', result etag = '{1}', saved to db." .format(fname, etag)) except: logger.error( "An exception occurred during upload of file or db update for '{0}'" .format(fname)) raise # output the "after" state results = db.fetchall( """SELECT id,file_harvest_date,file_harvest_etag FROM recordsets WHERE uuid=%s""", (rsuuid, )) for each in results: logger.debug("{0}".format(each)) return True