def main():
import argparse
from idb.config import config
parser = argparse.ArgumentParser(
description='Generate a report of newly discovered recordsets.')
parser.add_argument(
'-a',
'--age',
help=
'Number of days ago from today to mark beginning of "recent" period.',
default=31,
type=int)
parser.add_argument('-w',
'--write',
action='store_true',
help='Write to a File instead of STDOUT.')
args = parser.parse_args()
db = PostgresDB()
sql = """
select uuid, name, publisher_uuid, file_link, to_json(first_seen),
to_json(pub_date), to_json(file_harvest_date) from recordsets where ingest=false and
first_seen > now()-'%s days'::interval order by first_seen""" % args.age
results_list = db.fetchall(sql)
if args.write:
fh = open("fresh-recordsets-report.tsv", "w")
output_tsv(results_list, fh)
fh.close()
else:
output_tsv(results_list, sys.stdout)
def upload_recordset_from_file(rsid, fname):
"""
Given a recordset uuid and a local dataset filename, upload the local
dataset file as the "current" file for that uuid.
Parameters
----------
rsid : uuid
An iDigBio recordset uuid
fname : string
Filename (full path or current directory only)
Returns
-------
bool
True if successful, False otherwise
"""
# convert rsid uuid to string here because of either:
# psycopg2.ProgrammingError: can't adapt type 'UUID'
# or
# TypeError: 'UUID' object does not support indexing
rsuuid = str(rsid)
logger.info("Manual upload of '{0}' from file '{1}' requested.".format(
rsuuid, fname))
# do some checks here
try:
f = open(fname)
f.close()
except:
logger.error(
"Cannot access file: '{0}'. Aborting upload.".format(fname))
raise
db = PostgresDB()
sql = ("""SELECT id FROM recordsets WHERE uuid=%s""", (rsuuid, ))
idcount = db.execute(*sql)
if idcount < 1:
logger.error(
"Cannot find uuid '{0}' in db. Aborting upload.".format(rsuuid))
db.rollback()
return False
# output the "before" state
results = db.fetchall(
"""SELECT id,file_harvest_date,file_harvest_etag FROM recordsets WHERE uuid=%s""",
(rsuuid, ))
for each in results:
logger.debug("{0}".format(each))
try:
etag = upload_recordset(rsuuid, fname, db)
assert etag
sql = ("""UPDATE recordsets
SET file_harvest_etag=%s, file_harvest_date=%s
WHERE uuid=%s""", (etag, datetime.datetime.now(), rsuuid))
update_count = db.execute(*sql)
db.commit()
logger.info("UPDATED {0} rows.".format(update_count))
logger.info(
"Finished manual upload of file '{0}', result etag = '{1}', saved to db."
.format(fname, etag))
except:
logger.error(
"An exception occurred during upload of file or db update for '{0}'"
.format(fname))
raise
# output the "after" state
results = db.fetchall(
"""SELECT id,file_harvest_date,file_harvest_etag FROM recordsets WHERE uuid=%s""",
(rsuuid, ))
for each in results:
logger.debug("{0}".format(each))
return True