def test_valid_miseq_with_status_file_already_uploaded(self): """ Test a valid miseq directory for upload from end to end We create a status file that indicates the files have already been uploaded, Then make sure it does not upload :return: """ # Set our sample config file to use miseq parser and the correct irida credentials self.write_to_config_file( client_id=tests_integration.client_id, client_secret=tests_integration.client_secret, username=tests_integration.username, password=tests_integration.password, base_url=tests_integration.base_url, parser="miseq") # Write a status file to the upload directory directory_status = model.DirectoryStatus( directory=path.join(path_to_module, "fake_ngs_data")) directory_status.status = model.DirectoryStatus.COMPLETE progress.write_directory_status(directory_status) # Do the upload, without force option upload_result = upload_run_single_entry( path.join(path_to_module, "fake_ngs_data"), False) # Make sure the upload was a failure self.assertEqual(upload_result.exit_code, 1)
def test_write_to_new_file(self): # File does not exist yet self.assertFalse(path.exists(self.status_file)) # Create DirectoryStatus to use for writing directory_status = DirectoryStatus(self.directory) # Write to file directory_status.status = DirectoryStatus.COMPLETE progress.write_directory_status(directory_status) # Check that file matches what we wrote status = progress.get_directory_status(self.directory, ["SampleSheet.csv"]) self.assertEqual(DirectoryStatus.COMPLETE, status.status)
def test_write_with_readonly_is_true_file(self): # set readonly to True in config config.set_config_options(readonly=True) # File does not exist yet self.assertFalse(path.exists(self.status_file)) # Create DirectoryStatus to use for writing directory_status = DirectoryStatus(self.directory) # Try write to file (WRITE COMPLETE) directory_status.status = DirectoryStatus.COMPLETE progress.write_directory_status(directory_status) # Check that file matches what we wrote # We should find it as NEW as we have readonly=True status = progress.get_directory_status(self.directory, ["SampleSheet.csv"]) self.assertEqual(DirectoryStatus.NEW, status.status)
def _set_and_write_directory_status(directory_status, status, message=None, run_id=None): """ Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory :param directory_status: DirectoryStatus object :param status: a valid DirectoryStatus status :param message: string :param run_id: optional, if provided, the run id and irida instance will be included when written :return: """ if config.read_config_option("readonly", bool, False) is False: directory_status.status = status directory_status.message = message progress.write_directory_status(directory_status, run_id)
def _set_and_write_directory_status(directory_status, status, message=None): """ Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory :param directory_status: DirectoryStatus object :param status: a valid DirectoryStatus status :param message: string :return: """ try: directory_status.status = status directory_status.message = message progress.write_directory_status(directory_status) except progress.exceptions.DirectoryError as e: logging.error( "ERROR! Error while trying to write status file to directory {} with error message: {}" "".format(e.directory, e.message)) raise e
def _set_and_write_directory_status(directory_status, status, message=None, run_id=None): """ Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory :param directory_status: DirectoryStatus object :param status: a valid DirectoryStatus status :param message: string :param run_id: optional, if provided, the run id and irida instance will be included when written :return: """ directory_status.status = status directory_status.message = message if run_id: progress.write_directory_status(directory_status, run_id) else: progress.write_directory_status(directory_status)
def test_valid_miseq_with_status_file_force(self): """ Test a valid miseq directory for upload from end to end We create a status file that indicates the files have already been uploaded, and then use the force option to upload anyways :return: """ # Set our sample config file to use miseq parser and the correct irida credentials self.write_to_config_file( client_id=tests_integration.client_id, client_secret=tests_integration.client_secret, username=tests_integration.username, password=tests_integration.password, base_url=tests_integration.base_url, parser="miseq") # instance an api test_api = api.ApiCalls(client_id=tests_integration.client_id, client_secret=tests_integration.client_secret, base_url=tests_integration.base_url, username=tests_integration.username, password=tests_integration.password) # Create a test project, the uploader does not make new projects on its own # so one must exist to upload samples into # This may not be the project that the files get uploaded to, # but one will be made in the case this is the only test being run project_name = "test_project_3" project_description = "test_project_description" project = model.Project(name=project_name, description=project_description) test_api.send_project(project) # We always upload to project "1" so that tests will be consistent no matter how many / which tests are run project_id = "1" # Write a status file to the upload directory that we can force past directory_status = model.DirectoryStatus( directory=path.join(path_to_module, "fake_ngs_data_force")) directory_status.status = model.DirectoryStatus.COMPLETE progress.write_directory_status(directory_status) # Do the upload, with force option upload_result = upload_run_single_entry( path.join(path_to_module, "fake_ngs_data_force"), True) # Make sure the upload was a success self.assertEqual(upload_result.exit_code, 0) # Verify the files were uploaded sample_list = test_api.get_samples(project_id) sample_1_found = False sample_2_found = False sample_3_found = False for sample in sample_list: if sample.sample_name in ["01-1111", "02-2222", "03-3333"]: if sample.sample_name == "01-1111": sample_1_found = True sequence_files = test_api.get_sequence_files( project_id, sample.sample_name) self.assertEqual(len(sequence_files), 2) res_sequence_file_names = [ sequence_files[0]['fileName'], sequence_files[1]['fileName'] ] expected_sequence_file_names = [ '01-1111_S1_L001_R1_001.fastq.gz', '01-1111_S1_L001_R2_001.fastq.gz' ] self.assertEqual(res_sequence_file_names.sort(), expected_sequence_file_names.sort()) elif sample.sample_name == "02-2222": sample_2_found = True sequence_files = test_api.get_sequence_files( project_id, sample.sample_name) self.assertEqual(len(sequence_files), 2) res_sequence_file_names = [ sequence_files[0]['fileName'], sequence_files[1]['fileName'] ] expected_sequence_file_names = [ '02-2222_S1_L001_R1_001.fastq.gz', '02-2222_S1_L001_R2_001.fastq.gz' ] self.assertEqual(res_sequence_file_names.sort(), expected_sequence_file_names.sort()) elif sample.sample_name == "03-3333": sample_3_found = True sequence_files = test_api.get_sequence_files( project_id, sample.sample_name) self.assertEqual(len(sequence_files), 2) res_sequence_file_names = [ sequence_files[0]['fileName'], sequence_files[1]['fileName'] ] expected_sequence_file_names = [ '03-3333_S1_L001_R1_001.fastq.gz', '03-3333_S1_L001_R2_001.fastq.gz' ] self.assertEqual(res_sequence_file_names.sort(), expected_sequence_file_names.sort()) self.assertEqual(sample_1_found, True) self.assertEqual(sample_2_found, True) self.assertEqual(sample_3_found, True)
def upload_sequencing_run(sequencing_run, directory_status, upload_mode, run_id=None): """ Handles uploading a sequencing run Expects api to have been set up Expects sequencing run to have been validated Expects sequencing run to be valid for upload Expects directory_status to be valid :param sequencing_run: run to upload :param directory_status: DirectoryStatus object to update as files get uploaded :param upload_mode: mode of upload :param run_id: Default None, when given, run_id will be used instead of generating a new run_id :return: """ # get api api_instance = _get_api_instance() # create a new seq run identifier if none is given if run_id is None: run_id = api_instance.create_seq_run( sequencing_run.metadata, sequencing_run.sequencing_run_type) logging.info( "Sequencing run id '{}' has been created for this upload.".format( run_id)) else: logging.info( "Using existing run id '{}' for this upload.".format(run_id)) # Update directory status file directory_status.run_id = run_id progress.write_directory_status(directory_status) try: # set seq run to upload api_instance.set_seq_run_uploading(run_id) # loop through projects for project in sequencing_run.project_list: # loop through samples for sample in project.sample_list: if sample.skip: logging.info( "Skipping Sample {} on Project {}, already uploaded." "".format(sample.sample_name, project.id)) else: logging.info("Uploading to Sample {} on Project {}".format( sample.sample_name, project.id)) # upload files api_instance.send_sequence_files( sequence_file=sample.sequence_file, sample_name=sample.sample_name, project_id=project.id, upload_id=run_id, upload_mode=upload_mode) # Update status file on progress # Skipped samples are set to uploaded too, s.t. if they are skipped, # and the upload fails and is continued again, they will be skipped again. directory_status.set_sample_uploaded( sample_name=sample.sample_name, project_id=project.id, uploaded=True) progress.write_directory_status(directory_status) # set seq run to complete api_instance.set_seq_run_complete(run_id) # set seq run to error if there is an error except api.exceptions.IridaConnectionError as e: logging.error( "Failed to upload SequencingRun, Could not connect to IRIDA") raise e except api.exceptions.IridaResourceError as e: logging.error( "Failed to upload SequencingRun, Could not access resources on IRIDA" ) api_instance.set_seq_run_error(run_id) raise e except api.exceptions.FileError as e: logging.error( "Failed to upload SequencingRun, Could not access files to upload to IRIDA" ) api_instance.set_seq_run_error(run_id) raise e