def test_move_filepaths_to_upload_folder(self):
# setting up test, done here as this is the only test that uses these
# files
fd, seqs_fp = mkstemp(suffix='_seqs.fastq')
close(fd)
study_id = 1
rd = RawData.create(2, [Study(study_id)], [(seqs_fp, 1)])
filepaths = rd.get_filepaths()
# deleting reference so we can directly call
# move_filepaths_to_upload_folder
for fid, _, _ in filepaths:
self.conn_handler.execute(
"DELETE FROM qiita.raw_filepath WHERE filepath_id=%s", (fid,))
# moving filepaths
move_filepaths_to_upload_folder(study_id, filepaths, self.conn_handler)
# check that they do not exist in the old path but do in the new one
path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id))
for _, fp, _ in filepaths:
self.assertFalse(exists(fp))
new_fp = join(path_for_removal, basename(fp).split('_', 1)[1])
self.assertTrue(exists(new_fp))
self.files_to_remove.append(new_fp)
def test_move_filepaths_to_upload_folder(self):
# setting up test, done here as this is the only test that uses these
# files
fd, seqs_fp = mkstemp(suffix='_seqs.fastq')
close(fd)
study_id = 1
rd = RawData.create(2, [Study(study_id)], [(seqs_fp, 1)])
filepaths = rd.get_filepaths()
# deleting reference so we can directly call
# move_filepaths_to_upload_folder
for fid, _, _ in filepaths:
self.conn_handler.execute(
"DELETE FROM qiita.raw_filepath WHERE filepath_id=%s", (fid, ))
# moving filepaths
move_filepaths_to_upload_folder(study_id, filepaths, self.conn_handler)
# check that they do not exist in the old path but do in the new one
path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id))
for _, fp, _ in filepaths:
self.assertFalse(exists(fp))
new_fp = join(path_for_removal, basename(fp).split('_', 1)[1])
self.assertTrue(exists(new_fp))
self.files_to_remove.append(new_fp)
def test_move_filepaths_to_upload_folder(self):
# setting up test, done here as this is the only test that uses these
# files
fd, seqs_fp = mkstemp(suffix="_seqs.fastq")
close(fd)
st = Study(1)
metadata_dict = {
"SKB8.640193": {
"center_name": "ANL",
"primer": "GTGCCAGCMGCCGCGGTAA",
"barcode": "GTCCGCAAGTTA",
"run_prefix": "s_G1_L001_sequences",
"platform": "ILLUMINA",
"library_construction_protocol": "AAAA",
"experiment_design_description": "BBBB",
}
}
metadata = pd.DataFrame.from_dict(metadata_dict, orient="index")
pt = PrepTemplate.create(metadata, Study(1), "16S")
rd = RawData.create(2, [pt], [(seqs_fp, 1)])
filepaths = rd.get_filepaths()
# deleting reference so we can directly call
# move_filepaths_to_upload_folder
for fid, _, _ in filepaths:
self.conn_handler.execute("DELETE FROM qiita.raw_filepath WHERE filepath_id=%s", (fid,))
# moving filepaths
move_filepaths_to_upload_folder(st.id, filepaths)
# check that they do not exist in the old path but do in the new one
path_for_removal = join(get_mountpoint("uploads")[0][1], str(st.id))
for _, fp, _ in filepaths:
self.assertFalse(exists(fp))
new_fp = join(path_for_removal, basename(fp).split("_", 1)[1])
self.assertTrue(exists(new_fp))
self.files_to_remove.append(new_fp)
END $do$;
--- Drop the study_raw__data table as it's not longer used
DROP TABLE qiita.study_raw_data;
-- The raw_data_id column now can be nullable
ALTER TABLE qiita.prep_template
ALTER COLUMN raw_data_id DROP NOT NULL;
"""
conn_handler.add_to_queue(queue, sql)
conn_handler.execute_queue(queue)
# After the changes in the database have been performed, move the files
# to the uploads folder
errors = []
for st_id, fps in move_files:
try:
move_filepaths_to_upload_folder(st_id, fps)
except Exception, e:
# An error here is unlikely. However, it's possible and there is no
# clean way that we can unroll all the previous changes in the DB.
errors.append((st_id, fps, str(e)))
# Show the user any error that could have been generated during the files
# movement
if errors:
print ("The following errors where generated when trying to move files "
"to the upload folder")
for st_id, fps, e in errors:
print "Study: %d, Filepaths: %s, Error: %s" % (st_id, fps, e)
sql_delete = "DELETE FROM qiita.raw_data WHERE raw_data_id = %s"
sql_studies = """SELECT study_id FROM qiita.study_raw_data
WHERE raw_data_id = %s"""
move_files = []
for rd_id in rd_ids:
rd = RawData(rd_id)
filepaths = rd.get_filepaths()
TRN.add(sql_studies, [rd_id])
studies = TRN.execute_fetchflatten()
if filepaths:
# we need to move the files to a study. We chose the one with lower
# study id. Currently there is no case in the live database in
# which a RawData with no prep templates is attached to more than
# one study, but I think it is better to normalize this just
# in case
move_filepaths_to_upload_folder(min(studies), filepaths)
# To delete the RawData we first need to unlink all the files
TRN.add(sql_unlink, [rd_id])
# Then, remove the raw data from all the studies
for st_id in studies:
TRN.add(sql_detach, [rd_id, st_id])
TRN.add(sql_delete, [rd_id])
# We can now perform all changes in the DB. Although these changes can be
# done in an SQL patch, they are done here because we need to execute the
# previous clean up in the database before we can actually execute the SQL
# patch.
sql = """CREATE TABLE qiita.study_prep_template (
