def test_send_and_get_sample(self):
"""
Tests sending and receiving sample data
:return:
"""
# set up a project to upload samples to
project_name = "test_project_2"
project_description = "test_project_description"
project = model.Project(name=project_name, description=project_description)
proj_json_res = self.test_api.send_project(project)
project_identifier = proj_json_res['resource']['identifier']
# upload a sample
sample_name = "test_sample"
sample_desc = "test_sample_desc"
sample = model.Sample(sample_name, sample_desc)
sample_json_res = self.test_api.send_sample(sample, project_identifier)
# make sure the returned values match what we tried to upload
self.assertEqual(sample_json_res['resource']['sampleName'], sample_name)
self.assertEqual(sample_json_res['resource']['description'], sample_desc)
# get a list of samples on our project and make sure they match what we uploaded
sample_list = self.test_api.get_samples(project_identifier)
self.assertEqual(len(sample_list), 1)
self.assertEqual(type(sample_list[0]), model.Sample)
self.assertEqual(sample_list[0].sample_name, sample_name)
self.assertEqual(sample_list[0].description, sample_desc)
def build_sequencing_run_from_samples(sample_sheet_file, metadata,
sequencing_run_type):
"""
Create a SequencingRun object with full project/sample/sequence_file structure
:param sample_sheet_file:
:param metadata:
:param sequencing_run_type:
:return: SequencingRun
"""
sample_list = _parse_sample_list(sample_sheet_file)
logging.debug("Building SequencingRun from parsed data")
# create list of projects and add samples to appropriate project
project_list = []
for sample in sample_list:
project = None
for p in project_list:
if sample.get('sample_project') == p.id:
project = p
if project is None:
project = model.Project(id=sample.get('sample_project'))
project_list.append(project)
project.add_sample(sample)
sequence_run = model.SequencingRun(metadata=metadata,
project_list=project_list,
sequencing_run_type=sequencing_run_type)
logging.debug("SequencingRun built")
return sequence_run
def test_send_and_get_project(self):
"""
Tests sending and receiving project data
:return:
"""
# try sending a project
project_name = "test_project"
project_description = "test_project_description"
project = model.Project(name=project_name, description=project_description)
json_res = self.test_api.send_project(project)
# make sure the json response is what we expect it to be
self.assertEqual(json_res['resource']['name'], project_name)
self.assertEqual(json_res['resource']['projectDescription'], project_description)
# get a list of all projects
proj_list = self.test_api.get_projects()
proj = proj_list[len(proj_list) - 1]
# verify info matches what we uploaded
self.assertEqual(type(proj), model.Project)
self.assertEqual(proj.name, project_name)
self.assertEqual(proj.description, project_description)
# make sure identifiers match
self.assertEqual(json_res['resource']['identifier'], str(len(proj_list)))
self.assertEqual(json_res['resource']['identifier'], proj.id)
def _make_seq_run():
"""
Make a sequencing run pointed at real data for the tests
:return: SequencingRun object
"""
files_1 = model.SequenceFile([
path.join(path_to_module, "fake_ngs_data", "Data", "Intensities",
"BaseCalls", "01-1111_S1_L001_R1_001.fastq.gz"),
path.join(path_to_module, "fake_ngs_data", "Data", "Intensities",
"BaseCalls", "01-1111_S1_L001_R2_001.fastq.gz"),
])
files_2 = model.SequenceFile([
path.join(path_to_module, "fake_ngs_data", "Data", "Intensities",
"BaseCalls", "02-2222_S1_L001_R1_001.fastq.gz"),
path.join(path_to_module, "fake_ngs_data", "Data", "Intensities",
"BaseCalls", "02-2222_S1_L001_R2_001.fastq.gz"),
])
files_3 = model.SequenceFile([
path.join(path_to_module, "fake_ngs_data", "Data", "Intensities",
"BaseCalls", "03-3333_S1_L001_R1_001.fastq.gz"),
path.join(path_to_module, "fake_ngs_data", "Data", "Intensities",
"BaseCalls", "03-3333_S1_L001_R2_001.fastq.gz"),
])
sample_1 = model.Sample("test_sample", "description", 1)
sample_1.sequence_file = files_1
sample_2 = model.Sample("test_sample", "description", 1)
sample_2.sequence_file = files_2
sample_3 = model.Sample("test_sample", "description", 1)
sample_3.sequence_file = files_3
project = model.Project("test_project", [sample_1, sample_2, sample_3],
"description")
sequencing_run = model.SequencingRun({"layoutType": "PAIRED_END"},
[project], "miseq")
return sequencing_run
def build_sequencing_run_from_samples(sample_list, metadata,
sequence_run_type):
"""
Create a SequencingRun object with full project/sample/sequence_file structure
:param sample_list: List of Sample objects
:param metadata: metadata dict to add to the run
:param sequence_run_type: string identifier for sequencer type when generating api route
:return: SequencingRun
"""
logging.debug("Building SequencingRun from parsed data")
# create list of projects and add samples to appropriate project
project_list = []
for sample in sample_list:
project = None
for p in project_list:
if sample.get('sample_project') == p.id:
project = p
if project is None:
project = model.Project(id=sample.get('sample_project'))
project_list.append(project)
project.add_sample(sample)
sequence_run = model.SequencingRun(metadata=metadata,
project_list=project_list,
sequencing_run_type=sequence_run_type)
logging.debug("SequencingRun built")
return sequence_run
def test_project_exists(self):
"""
upload a project and check if project can be found with the projects_exists method
:return:
"""
project_name = "test_project_exists"
project_description = "test_project_exists_description"
project = model.Project(name=project_name, description=project_description)
json_res = self.test_api.send_project(project)
project_id = json_res['resource']['identifier']
self.assertTrue(self.test_api.project_exists(project_id))
def test_valid_assemblies_directory_upload(self):
"""
Test a valid directory of assemblies for upload end to end
:return:
"""
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="directory")
# instance an api
test_api = api.ApiCalls(client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
base_url=tests_integration.base_url,
username=tests_integration.username,
password=tests_integration.password)
# Create a test project, the uploader does not make new projects on its own
# so one must exist to upload samples into
# This may not be the project that the files get uploaded to,
# but one will be made in the case this is the only test being run
project_name = "test_project_assemblies"
project_description = "test_project_description_assemblies"
project = model.Project(name=project_name,
description=project_description)
test_api.send_project(project)
# We always upload to project "1" so that tests will be consistent no matter how many / which tests are run
project_id = "1"
# Do the upload
upload_result = upload_run_single_entry(directory=path.join(
path_to_module, "fake_assemblies_data"),
upload_assemblies=True)
# Make sure the upload was a success
self.assertEqual(upload_result.exit_code, 0)
# Verify the files were uploaded
sample_list = test_api.get_samples(project_id)
test_sample = sample_list[0]
sequence_files = test_api.get_assemblies_files(project_id,
test_sample.sample_name)
self.assertEqual(len(sequence_files), 1)
self.assertEqual(sequence_files[0]['fileName'], 'file_1.fasta')
def test_upload_to_nonexistent_project(self):
"""
Everything is correct except the sample sheet file specifies an invalid project
Samples should not be uploaded
:return:
"""
# try to upload to a non existent project
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="miseq",
readonly=False)
# instance an api
test_api = api.ApiCalls(client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
base_url=tests_integration.base_url,
username=tests_integration.username,
password=tests_integration.password)
# Create a test project, the uploader does not make new projects on its own
# so one must exist to upload samples into
# This may not be the project that the files get uploaded to,
# but one will be made in the case this is the only test being run
project_name = "test_project"
project_description = "test_project_description"
project = model.Project(name=project_name,
description=project_description)
test_api.send_project(project)
# Do the upload
upload_result = upload_run_single_entry(
path.join(path_to_module, "fake_ngs_data_nonexistent_project"))
# Make sure the upload was a failure
self.assertEqual(upload_result.exit_code, 1)
# Verify that the project does not exist
project_id = "1000"
with self.assertRaises(api.exceptions.IridaKeyError):
test_api.get_samples(project_id)
def test_send_and_get_sequence_files(self):
"""
Tests sending and receiving sequence files
:return:
"""
# upload a project
project_name = "test_project_2"
project_description = "test_project_description"
project = model.Project(name=project_name,
description=project_description)
proj_json_res = self.test_api.send_project(project)
project_identifier = proj_json_res['resource']['identifier']
# upload a sample
sample_name = "test_sample"
sample_desc = "test_sample_desc"
sample = model.Sample(sample_name, sample_desc)
self.test_api.send_sample(sample, project_identifier)
# upload sequence files
sequence_file_list = [
path.join(path_to_module, "fake_dir_data", "file_1.fastq.gz"),
path.join(path_to_module, "fake_dir_data", "file_2.fastq.gz")
]
sequence_file = model.SequenceFile(sequence_file_list)
upload_id = self.test_api.create_seq_run({'layoutType': 'PAIRED_END'},
'miseq')
self.test_api.send_sequence_files(sequence_file, sample_name,
project_identifier, upload_id)
# verify sequence files match what we sent to IRIDA
returned_sequence_files = self.test_api.get_sequence_files(
project_identifier, sample_name)
self.assertEqual(returned_sequence_files[0]['fileName'],
'file_1.fastq.gz')
self.assertEqual(returned_sequence_files[1]['fileName'],
'file_2.fastq.gz')
def test_sample_exists(self):
"""
Upload a sample and make sure it can be found with the sample_exists method
:return:
"""
# create a project to upload samples to
project_name = "test_project_exists"
project_description = "test_project_exists_description"
project = model.Project(name=project_name, description=project_description)
json_res = self.test_api.send_project(project)
project_id = json_res['resource']['identifier']
# create and upload a sample, and verify it exists
sample_name = "test_sample_exists"
sample_desc = "test_sample_exists_desc"
sample = model.Sample(sample_name, sample_desc)
self.test_api.send_sample(sample, project_id)
self.assertTrue(self.test_api.sample_exists(sample_name, project_id))
def test_invalid_read_only_miseq_upload(self):
"""
Test failing to upload a readonly miseq directory because readonly is turned off
:return:
"""
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="miseq",
readonly=False)
# instance an api
test_api = api.ApiCalls(client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
base_url=tests_integration.base_url,
username=tests_integration.username,
password=tests_integration.password)
# Create a test project, the uploader does not make new projects on its own
# so one must exist to upload samples into
# This may not be the project that the files get uploaded to,
# but one will be made in the case this is the only test being run
project_name = "test_read_only_project_fail"
project_description = "test_project_description"
project = model.Project(name=project_name,
description=project_description)
test_api.send_project(project)
# try the upload
upload_result = upload_run_single_entry(path.join(
path_to_module, "fake_ngs_data_read_only"),
force_upload=False,
upload_mode=api.MODE_DEFAULT)
# Make sure the upload was a failure
self.assertEqual(upload_result.exit_code, 1)
def test_upload_parse_fail(self):
"""
Given an invalid sample sheet, make sure that the upload does not happen
:return:
"""
# try to upload to a non existent project
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="miseq")
# instance an api
test_api = api.ApiCalls(client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
base_url=tests_integration.base_url,
username=tests_integration.username,
password=tests_integration.password)
# Create a test project, the uploader does not make new projects on its own
# so one must exist to upload samples into
# This may not be the project that the files get uploaded to,
# but one will be made in the case this is the only test being run
project_name = "test_project"
project_description = "test_project_description"
project = model.Project(name=project_name,
description=project_description)
test_api.send_project(project)
# Do the upload
upload_result = upload_run_single_entry(
path.join(path_to_module, "fake_ngs_data_parse_fail"))
# Make sure the upload was a failure
self.assertEqual(upload_result.exit_code, 1)
def get_projects(self):
"""
API call to api/projects to get list of projects
returns list containing projects. each project is Project object.
"""
logging.info("Loading projects.")
if self.cached_projects is None:
logging.debug("Loading projects from IRIDA server.")
url = self._get_link(self.base_url, "projects")
response = self._session.get(url)
result = response.json()["resource"]["resources"]
try:
project_list = [
model.Project(
name=project_dict["name"],
description=project_dict["projectDescription"],
id=project_dict["identifier"])
for project_dict in result
]
except KeyError as e:
e.args = map(str, e.args)
msg_arg = " ".join(e.args)
logging.debug(msg_arg + " not found. Available keys: "
", ".join(result[0].keys()))
raise exceptions.IridaKeyError(msg_arg +
" not found. Available keys: "
", ".join(result[0].keys()))
self.cached_projects = project_list
else:
logging.debug("Loading projects from cache.")
return self.cached_projects
def test_valid_miseq_upload(self):
"""
Test a valid miseq directory for upload from end to end
:return:
"""
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="miseq")
# instance an api
test_api = api.ApiCalls(client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
base_url=tests_integration.base_url,
username=tests_integration.username,
password=tests_integration.password)
# Create a test project, the uploader does not make new projects on its own
# so one must exist to upload samples into
# This may not be the project that the files get uploaded to,
# but one will be made in the case this is the only test being run
project_name = "test_project"
project_description = "test_project_description"
project = model.Project(name=project_name,
description=project_description)
test_api.send_project(project)
# We always upload to project "1" so that tests will be consistent no matter how many / which tests are run
project_id = "1"
# Do the upload
upload_result = upload_run_single_entry(
path.join(path_to_module, "fake_ngs_data"))
# Make sure the upload was a success
self.assertEqual(upload_result.exit_code, 0)
# Verify the files were uploaded
sample_list = test_api.get_samples(project_id)
sample_1_found = False
sample_2_found = False
sample_3_found = False
for sample in sample_list:
if sample.sample_name in ["01-1111", "02-2222", "03-3333"]:
if sample.sample_name == "01-1111":
sample_1_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
res_sequence_file_names = [
sequence_files[0]['fileName'],
sequence_files[1]['fileName']
]
expected_sequence_file_names = [
'01-1111_S1_L001_R1_001.fastq.gz',
'01-1111_S1_L001_R2_001.fastq.gz'
]
self.assertEqual(res_sequence_file_names.sort(),
expected_sequence_file_names.sort())
elif sample.sample_name == "02-2222":
sample_2_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
res_sequence_file_names = [
sequence_files[0]['fileName'],
sequence_files[1]['fileName']
]
expected_sequence_file_names = [
'02-2222_S1_L001_R1_001.fastq.gz',
'02-2222_S1_L001_R2_001.fastq.gz'
]
self.assertEqual(res_sequence_file_names.sort(),
expected_sequence_file_names.sort())
elif sample.sample_name == "03-3333":
sample_3_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
res_sequence_file_names = [
sequence_files[0]['fileName'],
sequence_files[1]['fileName']
]
expected_sequence_file_names = [
'03-3333_S1_L001_R1_001.fastq.gz',
'03-3333_S1_L001_R2_001.fastq.gz'
]
self.assertEqual(res_sequence_file_names.sort(),
expected_sequence_file_names.sort())
self.assertEqual(sample_1_found, True)
self.assertEqual(sample_2_found, True)
self.assertEqual(sample_3_found, True)
def test_valid_directory_upload(self):
"""
Test a valid directory for upload end to end
:return:
"""
# Set our sample config file to use miseq parser and the correct irida credentials
self.write_to_config_file(
client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
username=tests_integration.username,
password=tests_integration.password,
base_url=tests_integration.base_url,
parser="directory",
readonly=False)
# instance an api
test_api = api.ApiCalls(client_id=tests_integration.client_id,
client_secret=tests_integration.client_secret,
base_url=tests_integration.base_url,
username=tests_integration.username,
password=tests_integration.password)
# Create a test project, the uploader does not make new projects on its own
# so one must exist to upload samples into
# This may not be the project that the files get uploaded to,
# but one will be made in the case this is the only test being run
project_name = "test_project_2"
project_description = "test_project_description_2"
project = model.Project(name=project_name,
description=project_description)
test_api.send_project(project)
# We always upload to project "1" so that tests will be consistent no matter how many / which tests are run
project_id = "1"
# Do the upload
upload_result = upload_run_single_entry(
path.join(path_to_module, "fake_dir_data"))
# Make sure the upload was a success
self.assertEqual(upload_result.exit_code, 0)
# Verify the files were uploaded
sample_list = test_api.get_samples(project_id)
sample_1_found = False
sample_2_found = False
sample_3_found = False
for sample in sample_list:
if sample.sample_name in [
"my-sample-1", "my-sample-2", "my-sample-3"
]:
if sample.sample_name == "my-sample-1":
sample_1_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
self.assertEqual(sequence_files[0]['fileName'],
'file_1.fastq.gz')
self.assertEqual(sequence_files[1]['fileName'],
'file_2.fastq.gz')
elif sample.sample_name == "my-sample-2":
sample_2_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
self.assertEqual(sequence_files[0]['fileName'],
'samp_F.fastq.gz')
self.assertEqual(sequence_files[1]['fileName'],
'samp_R.fastq.gz')
elif sample.sample_name == "my-sample-3":
sample_3_found = True
sequence_files = test_api.get_sequence_files(
project_id, sample.sample_name)
self.assertEqual(len(sequence_files), 2)
self.assertEqual(sequence_files[0]['fileName'],
'germ_f.fastq.gz')
self.assertEqual(sequence_files[1]['fileName'],
'germ_r.fastq.gz')
self.assertEqual(sample_1_found, True)
self.assertEqual(sample_2_found, True)
self.assertEqual(sample_3_found, True)
