def test_move_filepaths_to_upload_folder(self): # setting up test, done here as this is the only test that uses these # files fd, seqs_fp = mkstemp(suffix='_seqs.fastq') close(fd) study_id = 1 rd = RawData.create(2, [Study(study_id)], [(seqs_fp, 1)]) filepaths = rd.get_filepaths() # deleting reference so we can directly call # move_filepaths_to_upload_folder for fid, _, _ in filepaths: self.conn_handler.execute( "DELETE FROM qiita.raw_filepath WHERE filepath_id=%s", (fid,)) # moving filepaths move_filepaths_to_upload_folder(study_id, filepaths, self.conn_handler) # check that they do not exist in the old path but do in the new one path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id)) for _, fp, _ in filepaths: self.assertFalse(exists(fp)) new_fp = join(path_for_removal, basename(fp).split('_', 1)[1]) self.assertTrue(exists(new_fp)) self.files_to_remove.append(new_fp)
def test_move_filepaths_to_upload_folder(self): # setting up test, done here as this is the only test that uses these # files fd, seqs_fp = mkstemp(suffix='_seqs.fastq') close(fd) study_id = 1 rd = RawData.create(2, [Study(study_id)], [(seqs_fp, 1)]) filepaths = rd.get_filepaths() # deleting reference so we can directly call # move_filepaths_to_upload_folder for fid, _, _ in filepaths: self.conn_handler.execute( "DELETE FROM qiita.raw_filepath WHERE filepath_id=%s", (fid, )) # moving filepaths move_filepaths_to_upload_folder(study_id, filepaths, self.conn_handler) # check that they do not exist in the old path but do in the new one path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id)) for _, fp, _ in filepaths: self.assertFalse(exists(fp)) new_fp = join(path_for_removal, basename(fp).split('_', 1)[1]) self.assertTrue(exists(new_fp)) self.files_to_remove.append(new_fp)
def test_move_filepaths_to_upload_folder(self): # setting up test, done here as this is the only test that uses these # files fd, seqs_fp = mkstemp(suffix="_seqs.fastq") close(fd) st = Study(1) metadata_dict = { "SKB8.640193": { "center_name": "ANL", "primer": "GTGCCAGCMGCCGCGGTAA", "barcode": "GTCCGCAAGTTA", "run_prefix": "s_G1_L001_sequences", "platform": "ILLUMINA", "library_construction_protocol": "AAAA", "experiment_design_description": "BBBB", } } metadata = pd.DataFrame.from_dict(metadata_dict, orient="index") pt = PrepTemplate.create(metadata, Study(1), "16S") rd = RawData.create(2, [pt], [(seqs_fp, 1)]) filepaths = rd.get_filepaths() # deleting reference so we can directly call # move_filepaths_to_upload_folder for fid, _, _ in filepaths: self.conn_handler.execute("DELETE FROM qiita.raw_filepath WHERE filepath_id=%s", (fid,)) # moving filepaths move_filepaths_to_upload_folder(st.id, filepaths) # check that they do not exist in the old path but do in the new one path_for_removal = join(get_mountpoint("uploads")[0][1], str(st.id)) for _, fp, _ in filepaths: self.assertFalse(exists(fp)) new_fp = join(path_for_removal, basename(fp).split("_", 1)[1]) self.assertTrue(exists(new_fp)) self.files_to_remove.append(new_fp)
END $do$; --- Drop the study_raw__data table as it's not longer used DROP TABLE qiita.study_raw_data; -- The raw_data_id column now can be nullable ALTER TABLE qiita.prep_template ALTER COLUMN raw_data_id DROP NOT NULL; """ conn_handler.add_to_queue(queue, sql) conn_handler.execute_queue(queue) # After the changes in the database have been performed, move the files # to the uploads folder errors = [] for st_id, fps in move_files: try: move_filepaths_to_upload_folder(st_id, fps) except Exception, e: # An error here is unlikely. However, it's possible and there is no # clean way that we can unroll all the previous changes in the DB. errors.append((st_id, fps, str(e))) # Show the user any error that could have been generated during the files # movement if errors: print ("The following errors where generated when trying to move files " "to the upload folder") for st_id, fps, e in errors: print "Study: %d, Filepaths: %s, Error: %s" % (st_id, fps, e)
sql_delete = "DELETE FROM qiita.raw_data WHERE raw_data_id = %s" sql_studies = """SELECT study_id FROM qiita.study_raw_data WHERE raw_data_id = %s""" move_files = [] for rd_id in rd_ids: rd = RawData(rd_id) filepaths = rd.get_filepaths() TRN.add(sql_studies, [rd_id]) studies = TRN.execute_fetchflatten() if filepaths: # we need to move the files to a study. We chose the one with lower # study id. Currently there is no case in the live database in # which a RawData with no prep templates is attached to more than # one study, but I think it is better to normalize this just # in case move_filepaths_to_upload_folder(min(studies), filepaths) # To delete the RawData we first need to unlink all the files TRN.add(sql_unlink, [rd_id]) # Then, remove the raw data from all the studies for st_id in studies: TRN.add(sql_detach, [rd_id, st_id]) TRN.add(sql_delete, [rd_id]) # We can now perform all changes in the DB. Although these changes can be # done in an SQL patch, they are done here because we need to execute the # previous clean up in the database before we can actually execute the SQL # patch. sql = """CREATE TABLE qiita.study_prep_template (