def upload(table_name, document_file):
"""
Upload documents into a receiving table.
<table> must be the name of a table in the receiving schema which has a
"document" column. All other columns in the table must be optional on
insert, as only "document" is provided.
<documents.ndjson> must be a newline-delimited JSON file containing one
document per line to insert as a table row.
"""
db = DatabaseSession()
try:
LOG.info(f"Copying documents from {document_file.name}")
row_count = db.copy_from_ndjson(("receiving", table_name, "document"), document_file)
LOG.info(f"Received {row_count:,} {table_name} records")
LOG.info("Committing all changes")
db.commit()
except:
LOG.info("Rolling back all changes; the database will not be modified")
db.rollback()
raise
def upload(manifest_file):
"""
Upload manifest records into the database receiving area.
<manifest.ndjson> must be a newline-delimited JSON file produced by this
command's sibling commands.
Once records are uploaded, the manifest ETL routine will reconcile the
manifest records with known identifiers and existing samples.
"""
db = DatabaseSession()
try:
LOG.info(f"Copying sample manifest records from {manifest_file.path}")
row_count = db.copy_from_ndjson(("receiving", "manifest", "document"), manifest_file)
LOG.info(f"Received {row_count:,} manifest records")
LOG.info("Committing all changes")
db.commit()
except:
LOG.info("Rolling back all changes; the database will not be modified")
db.rollback()
raise
def upload(consensus_genome_file):
"""
Upload consensus genomes and summary statistics to the warehouse receiving area.
Consensus genomes and summary statistics should be in newline-delimited JSON
format that matches those generated by the assembly pipeline.
"""
db = DatabaseSession()
try:
LOG.info(
f"Copying consensus genome records from {consensus_genome_file.name}"
)
row_count = db.copy_from_ndjson(
("receiving", "consensus_genome", "document"),
consensus_genome_file)
LOG.info(f"Received {row_count:,} consensus genome records")
LOG.info("Committing all changes")
db.commit()
except:
LOG.info("Rolling back all changes; the database will not be modified")
db.rollback()
raise
def upload(det_file):
"""
Upload REDCap DET notifications into database receiving area.
<det.ndjson> must be a newline-delimited JSON file produced by this
command's sibling command.
"""
db = DatabaseSession()
try:
LOG.info(f"Copying REDCap DET records from {det_file.name}")
row_count = db.copy_from_ndjson(
("receiving", "redcap_det", "document"), det_file)
LOG.info(f"Received {row_count:,} DET records")
LOG.info("Committing all changes")
db.commit()
except:
LOG.info("Rolling back all changes; the database will not be modified")
db.rollback()
raise