def test_run(self):
"""test run"""
archive_dir = "tmp.test_spreadsheet_importer_run.archive"
os.mkdir(archive_dir)
tsv_file = "tmp.test_spreadsheet_importer_run.out.tsv"
# need to copy the dropbox directory, because run() will move the
# xlsx file
original_dropbox_dir = os.path.join(data_dir, "run.dropbox")
dropbox_dir = "tmp.test_spreadsheet_importer_run.out.dropbox"
shutil.copytree(original_dropbox_dir, dropbox_dir)
xlsx_file = os.path.join(dropbox_dir, "import.xlsx")
date = utils.date_string_from_file_mtime(xlsx_file)
importer = spreadsheet_importer.SpreadsheetImporter(
dropbox_dir, xlsx_file, db_ini_file, archive_dir, tsv_file
)
# test lock file stops it running
utils.make_empty_file(importer.lock_file)
with self.assertRaises(Exception):
importer.run()
os.unlink(importer.lock_file)
# we'll just run it - the details are checked in test_import_reads_and_update_db
importer.run()
shutil.rmtree(archive_dir)
shutil.rmtree(dropbox_dir)
os.unlink(tsv_file)
def _archive_spreadsheet(cls, xlsx_file, archive_dir):
date_string = utils.date_string_from_file_mtime(xlsx_file)
date_directory = os.path.join(archive_dir, date_string)
if not os.path.exists(date_directory):
try:
os.mkdir(date_directory)
except:
raise Exception("Error mkdir " + date_directory)
xlsx_basename = os.path.basename(xlsx_file)
existing_xlsx_files = set(os.listdir(date_directory))
# this is unlikely to happen, but if name of archived xlsx
# file already exists, append .N on the end (where N is smallest
# int such that file does not already exist)
if xlsx_basename in existing_xlsx_files:
i = 1
while xlsx_basename + "." + str(i) in existing_xlsx_files:
i += 1
xlsx_basename += "." + str(i)
new_name = os.path.join(date_directory, xlsx_basename)
utils.rsync_and_md5(xlsx_file, new_name)
os.unlink(xlsx_file)
return new_name
def test_date_string_from_file_mtime(self):
'''test date_string_from_file_mtime'''
tmpfile = 'tmp.test_date_string_from_file_mtime'
with open(tmpfile, 'w'):
pass
today = datetime.date.today()
got = utils.date_string_from_file_mtime(tmpfile)
def int_to_str(x):
if x < 10:
return '0' + str(x)
else:
return str(x)
self.assertEqual(str(today.year), got[0:4])
self.assertEqual(int_to_str(today.month), got[4:6])
self.assertEqual(int_to_str(today.day), got[6:])
os.unlink(tmpfile)
def test_archive_spreadsheet(self):
"""test _archive_spreadsheet"""
archive_dir = "tmp.test_archive_spreadsheet_archive"
os.mkdir(archive_dir)
xlsx_file = "tmp.test_archive_spreadsheet.xlsx"
xlsx_file2 = "tmp.test_archive_spreadsheet2.xlsx"
with open(xlsx_file, "w"):
pass
date = utils.date_string_from_file_mtime(xlsx_file)
date_dir = os.path.join(archive_dir, date)
got_filename = spreadsheet_importer.SpreadsheetImporter._archive_spreadsheet(
xlsx_file, archive_dir
)
expected_filename = os.path.join(date_dir, xlsx_file)
self.assertEqual(expected_filename, got_filename)
self.assertFalse(os.path.exists(xlsx_file))
self.assertTrue(os.path.exists(expected_filename))
with open(xlsx_file, "w"):
pass
got_filename = spreadsheet_importer.SpreadsheetImporter._archive_spreadsheet(
xlsx_file, archive_dir
)
expected_filename = os.path.join(date_dir, xlsx_file + ".1")
self.assertEqual(expected_filename, got_filename)
self.assertFalse(os.path.exists(xlsx_file))
self.assertTrue(os.path.exists(expected_filename))
with open(xlsx_file2, "w"):
pass
got_filename = spreadsheet_importer.SpreadsheetImporter._archive_spreadsheet(
xlsx_file2, archive_dir
)
expected_filename = os.path.join(date_dir, xlsx_file2)
self.assertEqual(expected_filename, got_filename)
self.assertFalse(os.path.exists(xlsx_file2))
self.assertTrue(os.path.exists(expected_filename))
shutil.rmtree(archive_dir)
def test_archive_spreadsheet(self):
'''test _archive_spreadsheet'''
archive_dir = 'tmp.test_archive_spreadsheet_archive'
os.mkdir(archive_dir)
xlsx_file = 'tmp.test_archive_spreadsheet.xlsx'
xlsx_file2 = 'tmp.test_archive_spreadsheet2.xlsx'
with open(xlsx_file, 'w'):
pass
date = utils.date_string_from_file_mtime(xlsx_file)
date_dir = os.path.join(archive_dir, date)
got_filename = spreadsheet_importer.SpreadsheetImporter._archive_spreadsheet(
xlsx_file, archive_dir)
expected_filename = os.path.join(date_dir, xlsx_file)
self.assertEqual(expected_filename, got_filename)
self.assertFalse(os.path.exists(xlsx_file))
self.assertTrue(os.path.exists(expected_filename))
with open(xlsx_file, 'w'):
pass
got_filename = spreadsheet_importer.SpreadsheetImporter._archive_spreadsheet(
xlsx_file, archive_dir)
expected_filename = os.path.join(date_dir, xlsx_file + '.1')
self.assertEqual(expected_filename, got_filename)
self.assertFalse(os.path.exists(xlsx_file))
self.assertTrue(os.path.exists(expected_filename))
with open(xlsx_file2, 'w'):
pass
got_filename = spreadsheet_importer.SpreadsheetImporter._archive_spreadsheet(
xlsx_file2, archive_dir)
expected_filename = os.path.join(date_dir, xlsx_file2)
self.assertEqual(expected_filename, got_filename)
self.assertFalse(os.path.exists(xlsx_file2))
self.assertTrue(os.path.exists(expected_filename))
shutil.rmtree(archive_dir)
def test_import_reads_and_update_db(self):
"""test _import_reads_and_update_db"""
archive_dir = "tmp.test_spreadsheet_importer_run.archive"
os.mkdir(archive_dir)
db_backup_dir = "tmp.test_spreadsheet_importer_run.db_backup"
os.mkdir(db_backup_dir)
tsv_file = "tmp.test_spreadsheet_importer_run.out.tsv"
# need to copy the dropbox directory, because run() will move the
# xlsx file
original_dropbox_dir = os.path.join(data_dir, "run.dropbox")
dropbox_dir = "tmp.test_spreadsheet_importer_run.out.dropbox"
shutil.copytree(original_dropbox_dir, dropbox_dir)
xlsx_file = os.path.join(dropbox_dir, "import.xlsx")
date = utils.date_string_from_file_mtime(xlsx_file)
importer = spreadsheet_importer.SpreadsheetImporter(
dropbox_dir,
xlsx_file,
db_ini_file,
archive_dir,
tsv_file,
db_backup_dir=db_backup_dir,
)
importer._import_reads_and_update_db()
# check xlsx file and import jobs file got archived
xlsx_archive_file = os.path.join(archive_dir, date, "import.xlsx")
self.assertTrue(os.path.exists(xlsx_archive_file))
self.assertTrue(os.path.exists(xlsx_archive_file + ".import_jobs.tsv"))
self.assertFalse(os.path.exists(xlsx_file))
shutil.rmtree(archive_dir)
# check database got backed up
backup_files = os.listdir(db_backup_dir)
self.assertEqual(1, len(backup_files))
shutil.rmtree(db_backup_dir)
# check tsv file is correct
reads_prefix = os.path.abspath(os.path.join(dropbox_dir, "reads"))
expected_tsv_lines = [
"\t".join(
[
"seqrep_id",
"sample_id",
"isolate_id",
"sequence_replicate_number",
"reads1",
"reads2",
"reads1_md5",
"reads2_md5",
]
),
"\t".join(
[
"1",
"1",
"1",
"43",
reads_prefix + ".1_1.fq",
reads_prefix + ".1_2.fq",
"abcdefghijklmnopqrstuvwyx123456",
"abcdefghijklmnopqrstuvwyx123457",
]
),
"\t".join(
[
"2",
"2",
"2",
"45",
reads_prefix + ".2_1.fq",
reads_prefix + ".2_2.fq",
"a73817805eb1d44ca88eb5cb794c7de7",
"d468360c689d482b227256d887a05996",
]
),
]
with open(tsv_file) as f:
got_tsv_lines = [line.rstrip() for line in f]
self.assertEqual(expected_tsv_lines, got_tsv_lines)
os.unlink(tsv_file)
# check database is correct
got_sample = self.db.get_rows_from_table("Sample", order_by="sample_id")
expected_sample = [
{
"sample_id": 1,
"subject_id": "p1",
"site_id": "s1",
"sample_id_from_lab": "l1",
"dataset_name": "g1",
"ena_center_name": "Center 1",
"ena_sample_accession": "ERS123456",
"ena_study_accession": None,
},
{
"sample_id": 2,
"subject_id": "p2",
"site_id": "s2",
"sample_id_from_lab": "l2",
"dataset_name": "g2",
"ena_center_name": "Center 1",
"ena_sample_accession": None,
"ena_study_accession": None,
},
]
self.maxDiff = None
self.assertEqual(expected_sample, got_sample)
got_seqrep = self.db.get_rows_from_table("Seqrep", order_by="seqrep_id")
expected_seqrep = [
{
"seqrep_id": 1,
"isolate_id": 1,
"sequence_replicate_number": 43,
"original_reads_file_1_md5": "abcdefghijklmnopqrstuvwyx123456",
"original_reads_file_2_md5": "abcdefghijklmnopqrstuvwyx123457",
"remove_contam_reads_file_1_md5": None,
"remove_contam_reads_file_2_md5": None,
"withdrawn": 0,
"import_status": 0,
"submission_date": datetime.date(2017, 12, 25),
"instrument_model": "Illumina HiSeq 2000",
"submit_to_ena": 0,
"ena_run_accession": "ERR123456",
"ena_on_hold": 0,
},
{
"seqrep_id": 2,
"isolate_id": 2,
"sequence_replicate_number": 45,
"original_reads_file_1_md5": "a73817805eb1d44ca88eb5cb794c7de7",
"original_reads_file_2_md5": "d468360c689d482b227256d887a05996",
"remove_contam_reads_file_1_md5": None,
"remove_contam_reads_file_2_md5": None,
"withdrawn": 0,
"import_status": 0,
"submission_date": datetime.date(2017, 12, 26),
"instrument_model": "Illumina HiSeq 2000",
"submit_to_ena": 1,
"ena_run_accession": None,
"ena_on_hold": 1,
},
]
self.assertEqual(expected_seqrep, got_seqrep)
shutil.rmtree(dropbox_dir)
def test_import_reads_and_update_db(self):
'''test _import_reads_and_update_db'''
archive_dir = 'tmp.test_spreadsheet_importer_run.archive'
os.mkdir(archive_dir)
db_backup_dir = 'tmp.test_spreadsheet_importer_run.db_backup'
os.mkdir(db_backup_dir)
tsv_file = 'tmp.test_spreadsheet_importer_run.out.tsv'
# need to copy the dropbox directory, because run() will move the
# xlsx file
original_dropbox_dir = os.path.join(data_dir, 'run.dropbox')
dropbox_dir = 'tmp.test_spreadsheet_importer_run.out.dropbox'
shutil.copytree(original_dropbox_dir, dropbox_dir)
xlsx_file = os.path.join(dropbox_dir, 'import.xlsx')
date = utils.date_string_from_file_mtime(xlsx_file)
importer = spreadsheet_importer.SpreadsheetImporter(
dropbox_dir,
xlsx_file,
db_ini_file,
archive_dir,
tsv_file,
db_backup_dir=db_backup_dir)
importer._import_reads_and_update_db()
# check xlsx file and import jobs file got archived
xlsx_archive_file = os.path.join(archive_dir, date, 'import.xlsx')
self.assertTrue(os.path.exists(xlsx_archive_file))
self.assertTrue(os.path.exists(xlsx_archive_file + '.import_jobs.tsv'))
self.assertFalse(os.path.exists(xlsx_file))
shutil.rmtree(archive_dir)
# check database got backed up
backup_files = os.listdir(db_backup_dir)
self.assertEqual(1, len(backup_files))
shutil.rmtree(db_backup_dir)
# check tsv file is correct
reads_prefix = os.path.abspath(os.path.join(dropbox_dir, 'reads'))
expected_tsv_lines = [
'\t'.join([
'seqrep_id', 'sample_id', 'isolate_id',
'sequence_replicate_number', 'reads1', 'reads2', 'reads1_md5',
'reads2_md5'
]),
'\t'.join([
'1', '1', '1', '43', reads_prefix + '.1_1.fq',
reads_prefix + '.1_2.fq', 'abcdefghijklmnopqrstuvwyx123456',
'abcdefghijklmnopqrstuvwyx123457'
]),
'\t'.join([
'2', '2', '2', '45', reads_prefix + '.2_1.fq',
reads_prefix + '.2_2.fq', 'a73817805eb1d44ca88eb5cb794c7de7',
'd468360c689d482b227256d887a05996'
]),
]
with open(tsv_file) as f:
got_tsv_lines = [line.rstrip() for line in f]
self.assertEqual(expected_tsv_lines, got_tsv_lines)
os.unlink(tsv_file)
# check database is correct
got_sample = self.db.get_rows_from_table('Sample',
order_by='sample_id')
expected_sample = [{
'sample_id': 1,
'subject_id': 'p1',
'site_id': 's1',
'sample_id_from_lab': 'l1',
'dataset_name': 'g1',
'ena_center_name': 'Center 1',
'ena_sample_accession': 'ERS123456',
'ena_study_accession': None,
}, {
'sample_id': 2,
'subject_id': 'p2',
'site_id': 's2',
'sample_id_from_lab': 'l2',
'dataset_name': 'g2',
'ena_center_name': 'Center 1',
'ena_sample_accession': None,
'ena_study_accession': None,
}]
self.maxDiff = None
self.assertEqual(expected_sample, got_sample)
got_seqrep = self.db.get_rows_from_table('Seqrep',
order_by='seqrep_id')
expected_seqrep = [{
'seqrep_id': 1,
'isolate_id': 1,
'sequence_replicate_number': 43,
'original_reads_file_1_md5': 'abcdefghijklmnopqrstuvwyx123456',
'original_reads_file_2_md5': 'abcdefghijklmnopqrstuvwyx123457',
'remove_contam_reads_file_1_md5': None,
'remove_contam_reads_file_2_md5': None,
'withdrawn': 0,
'import_status': 0,
'submission_date': datetime.date(2017, 12, 25),
'instrument_model': 'Illumina HiSeq 2000',
'submit_to_ena': 0,
'ena_run_accession': 'ERR123456',
'ena_on_hold': 0,
}, {
'seqrep_id': 2,
'isolate_id': 2,
'sequence_replicate_number': 45,
'original_reads_file_1_md5': 'a73817805eb1d44ca88eb5cb794c7de7',
'original_reads_file_2_md5': 'd468360c689d482b227256d887a05996',
'remove_contam_reads_file_1_md5': None,
'remove_contam_reads_file_2_md5': None,
'withdrawn': 0,
'import_status': 0,
'submission_date': datetime.date(2017, 12, 26),
'instrument_model': 'Illumina HiSeq 2000',
'submit_to_ena': 1,
'ena_run_accession': None,
'ena_on_hold': 1,
}]
self.assertEqual(expected_seqrep, got_seqrep)
shutil.rmtree(dropbox_dir)