def test_valid_miseq_with_status_file_already_uploaded(self):
"""
Test a valid miseq directory for upload from end to end
We create a status file that indicates the files have already been uploaded,
Then make sure it does not upload
:return:
"""
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="miseq")
# Write a status file to the upload directory
directory_status = model.DirectoryStatus(
directory=path.join(path_to_module, "fake_ngs_data"))
directory_status.status = model.DirectoryStatus.COMPLETE
progress.write_directory_status(directory_status)
# Do the upload, without force option
upload_result = upload_run_single_entry(
path.join(path_to_module, "fake_ngs_data"), False)
# Make sure the upload was a failure
self.assertEqual(upload_result.exit_code, 1)
def test_write_to_new_file(self):
# File does not exist yet
self.assertFalse(path.exists(self.status_file))
# Create DirectoryStatus to use for writing
directory_status = DirectoryStatus(self.directory)
# Write to file
directory_status.status = DirectoryStatus.COMPLETE
progress.write_directory_status(directory_status)
# Check that file matches what we wrote
status = progress.get_directory_status(self.directory,
["SampleSheet.csv"])
self.assertEqual(DirectoryStatus.COMPLETE, status.status)
def test_write_with_readonly_is_true_file(self):
# set readonly to True in config
config.set_config_options(readonly=True)
# File does not exist yet
self.assertFalse(path.exists(self.status_file))
# Create DirectoryStatus to use for writing
directory_status = DirectoryStatus(self.directory)
# Try write to file (WRITE COMPLETE)
directory_status.status = DirectoryStatus.COMPLETE
progress.write_directory_status(directory_status)
# Check that file matches what we wrote
# We should find it as NEW as we have readonly=True
status = progress.get_directory_status(self.directory,
["SampleSheet.csv"])
self.assertEqual(DirectoryStatus.NEW, status.status)
def _set_and_write_directory_status(directory_status,
status,
message=None,
run_id=None):
"""
Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory
:param directory_status: DirectoryStatus object
:param status: a valid DirectoryStatus status
:param message: string
:param run_id: optional, if provided, the run id and irida instance will be included when written
:return:
"""
if config.read_config_option("readonly", bool, False) is False:
directory_status.status = status
directory_status.message = message
progress.write_directory_status(directory_status, run_id)
def _set_and_write_directory_status(directory_status, status, message=None):
"""
Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory
:param directory_status: DirectoryStatus object
:param status: a valid DirectoryStatus status
:param message: string
:return:
"""
try:
directory_status.status = status
directory_status.message = message
progress.write_directory_status(directory_status)
except progress.exceptions.DirectoryError as e:
logging.error(
"ERROR! Error while trying to write status file to directory {} with error message: {}"
"".format(e.directory, e.message))
raise e
def _set_and_write_directory_status(directory_status,
status,
message=None,
run_id=None):
"""
Given a DirectoryStatus object, sets the status and message, and then writes to the directory status directory
:param directory_status: DirectoryStatus object
:param status: a valid DirectoryStatus status
:param message: string
:param run_id: optional, if provided, the run id and irida instance will be included when written
:return:
"""
directory_status.status = status
directory_status.message = message
if run_id:
progress.write_directory_status(directory_status, run_id)
else:
progress.write_directory_status(directory_status)
def test_valid_miseq_with_status_file_force(self):
"""
Test a valid miseq directory for upload from end to end
We create a status file that indicates the files have already been uploaded,
and then use the force option to upload anyways
:return:
"""
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="miseq")
# instance an api
test_api = api.ApiCalls(client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
base_url=tests_integration.base_url,
username=tests_integration.username,
password=tests_integration.password)
# Create a test project, the uploader does not make new projects on its own
# so one must exist to upload samples into
# This may not be the project that the files get uploaded to,
# but one will be made in the case this is the only test being run
project_name = "test_project_3"
project_description = "test_project_description"
project = model.Project(name=project_name,
description=project_description)
test_api.send_project(project)
# We always upload to project "1" so that tests will be consistent no matter how many / which tests are run
project_id = "1"
# Write a status file to the upload directory that we can force past
directory_status = model.DirectoryStatus(
directory=path.join(path_to_module, "fake_ngs_data_force"))
directory_status.status = model.DirectoryStatus.COMPLETE
progress.write_directory_status(directory_status)
# Do the upload, with force option
upload_result = upload_run_single_entry(
path.join(path_to_module, "fake_ngs_data_force"), True)
# Make sure the upload was a success
self.assertEqual(upload_result.exit_code, 0)
# Verify the files were uploaded
sample_list = test_api.get_samples(project_id)
sample_1_found = False
sample_2_found = False
sample_3_found = False
for sample in sample_list:
if sample.sample_name in ["01-1111", "02-2222", "03-3333"]:
if sample.sample_name == "01-1111":
sample_1_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
res_sequence_file_names = [
sequence_files[0]['fileName'],
sequence_files[1]['fileName']
]
expected_sequence_file_names = [
'01-1111_S1_L001_R1_001.fastq.gz',
'01-1111_S1_L001_R2_001.fastq.gz'
]
self.assertEqual(res_sequence_file_names.sort(),
expected_sequence_file_names.sort())
elif sample.sample_name == "02-2222":
sample_2_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
res_sequence_file_names = [
sequence_files[0]['fileName'],
sequence_files[1]['fileName']
]
expected_sequence_file_names = [
'02-2222_S1_L001_R1_001.fastq.gz',
'02-2222_S1_L001_R2_001.fastq.gz'
]
self.assertEqual(res_sequence_file_names.sort(),
expected_sequence_file_names.sort())
elif sample.sample_name == "03-3333":
sample_3_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
res_sequence_file_names = [
sequence_files[0]['fileName'],
sequence_files[1]['fileName']
]
expected_sequence_file_names = [
'03-3333_S1_L001_R1_001.fastq.gz',
'03-3333_S1_L001_R2_001.fastq.gz'
]
self.assertEqual(res_sequence_file_names.sort(),
expected_sequence_file_names.sort())
self.assertEqual(sample_1_found, True)
self.assertEqual(sample_2_found, True)
self.assertEqual(sample_3_found, True)
def upload_sequencing_run(sequencing_run,
directory_status,
upload_mode,
run_id=None):
"""
Handles uploading a sequencing run
Expects api to have been set up
Expects sequencing run to have been validated
Expects sequencing run to be valid for upload
Expects directory_status to be valid
:param sequencing_run: run to upload
:param directory_status: DirectoryStatus object to update as files get uploaded
:param upload_mode: mode of upload
:param run_id: Default None, when given, run_id will be used instead of generating a new run_id
:return:
"""
# get api
api_instance = _get_api_instance()
# create a new seq run identifier if none is given
if run_id is None:
run_id = api_instance.create_seq_run(
sequencing_run.metadata, sequencing_run.sequencing_run_type)
logging.info(
"Sequencing run id '{}' has been created for this upload.".format(
run_id))
else:
logging.info(
"Using existing run id '{}' for this upload.".format(run_id))
# Update directory status file
directory_status.run_id = run_id
progress.write_directory_status(directory_status)
try:
# set seq run to upload
api_instance.set_seq_run_uploading(run_id)
# loop through projects
for project in sequencing_run.project_list:
# loop through samples
for sample in project.sample_list:
if sample.skip:
logging.info(
"Skipping Sample {} on Project {}, already uploaded."
"".format(sample.sample_name, project.id))
else:
logging.info("Uploading to Sample {} on Project {}".format(
sample.sample_name, project.id))
# upload files
api_instance.send_sequence_files(
sequence_file=sample.sequence_file,
sample_name=sample.sample_name,
project_id=project.id,
upload_id=run_id,
upload_mode=upload_mode)
# Update status file on progress
# Skipped samples are set to uploaded too, s.t. if they are skipped,
# and the upload fails and is continued again, they will be skipped again.
directory_status.set_sample_uploaded(
sample_name=sample.sample_name,
project_id=project.id,
uploaded=True)
progress.write_directory_status(directory_status)
# set seq run to complete
api_instance.set_seq_run_complete(run_id)
# set seq run to error if there is an error
except api.exceptions.IridaConnectionError as e:
logging.error(
"Failed to upload SequencingRun, Could not connect to IRIDA")
raise e
except api.exceptions.IridaResourceError as e:
logging.error(
"Failed to upload SequencingRun, Could not access resources on IRIDA"
)
api_instance.set_seq_run_error(run_id)
raise e
except api.exceptions.FileError as e:
logging.error(
"Failed to upload SequencingRun, Could not access files to upload to IRIDA"
)
api_instance.set_seq_run_error(run_id)
raise e