def test_paired_end_files():
upload_sequence(
("tests/data/files/test_R1_L001.fq.gz",
"tests/data/files/test_R2_L001.fq.gz"),
FakeProxySession(200),
FakeSamplesResource(),
)
def test_upload_lots_of_files(files, n_uploads, fxi_calls, fxp_calls, size_calls):
fake_size = lambda filename: int(float(filename.split(".")[1])) # noqa
uso = "onecodex.lib.upload.upload_sequence_fileobj"
udo = "onecodex.lib.upload.upload_document_fileobj"
fxi = "onecodex.lib.upload.FASTXInterleave"
fxp = "onecodex.lib.upload.FilePassthru"
sz = "onecodex.lib.upload.os.path.getsize"
with patch(uso) as upload_sequence_fileobj, patch(fxi) as interleave, patch(fxp) as passthru:
with patch(sz, size_effect=fake_size) as size:
upload_sequence(files, FakeSession(), FakeSamplesResource())
assert upload_sequence_fileobj.call_count == n_uploads
assert interleave.call_count == fxi_calls
assert passthru.call_count == fxp_calls
assert size.call_count == size_calls
with patch(udo) as upload_document_fileobj, patch(fxp) as passthru:
with patch(sz, size_effect=fake_size) as size:
files = files[0] if isinstance(files, tuple) else files
upload_document(files, FakeSession(), FakeDocumentsResource())
assert upload_document_fileobj.call_count == n_uploads
assert passthru.call_count == fxp_calls + fxi_calls
assert size.call_count == fxp_calls + fxi_calls
def test_upload_lots_of_files(files, n_uploads, fxi_calls, fxp_calls,
size_calls):
fake_size = lambda filename: int(float(filename.split(".")[1])) # noqa
uso = "onecodex.lib.upload.upload_sequence_fileobj"
udo = "onecodex.lib.upload.upload_document_fileobj"
fxi = "onecodex.lib.upload.FASTXInterleave"
fxp = "onecodex.lib.upload.FilePassthru"
sz = "onecodex.lib.upload.os.path.getsize"
with patch(uso) as upload_sequence_fileobj, patch(
fxi) as interleave, patch(fxp) as passthru:
with patch(sz, size_effect=fake_size) as size:
upload_sequence(files, FakeProxySession(200),
FakeSamplesResource())
assert upload_sequence_fileobj.call_count == n_uploads
assert interleave.call_count == fxi_calls
assert passthru.call_count == fxp_calls
assert size.call_count == size_calls
with patch(udo) as upload_document_fileobj, patch(fxp) as passthru:
with patch(sz, size_effect=fake_size) as size:
files = files[0] if isinstance(files, tuple) else files
upload_document(files, FakeDocumentsResource())
assert upload_document_fileobj.call_count == n_uploads
assert passthru.call_count == fxp_calls + fxi_calls
assert size.call_count == fxp_calls + fxi_calls
def test_upload_lots_of_files(files, n_uploads, fxi_calls, fxp_calls):
with patch("boto3.session.Session"):
fake_size = lambda filename: int(float(filename.split(".")[1])) # noqa
uso = "onecodex.lib.upload._upload_sequence_fileobj"
udo = "onecodex.lib.upload._upload_document_fileobj"
fxi = "onecodex.lib.files.PairedEndFiles"
fxp = "onecodex.lib.files.FilePassthru"
sz = "os.path.getsize"
sample_resource = FakeSamplesResource()
with patch(uso) as upload_sequence_fileobj, patch(
fxi) as paired, patch(fxp) as passthru:
with patch(sz, size_effect=fake_size):
upload_sequence(files, sample_resource)
assert upload_sequence_fileobj.call_count == n_uploads
assert paired.call_count == fxi_calls
assert passthru.call_count == fxp_calls
# Need to patch from `upload.` instead of `files.` for that test
with patch(udo) as upload_document_fileobj, patch(
"onecodex.lib.upload.FilePassthru") as passthru:
with patch(sz, size_effect=fake_size):
files = files[0] if isinstance(files, tuple) else files
upload_document(files, FakeDocumentsResource())
assert (upload_document_fileobj.call_count == n_uploads -
1 if isinstance(files, tuple) else n_uploads)
assert passthru.call_count == fxp_calls + fxi_calls
def test_paired_end_files():
with patch("boto3.session.Session") as b3:
upload_sequence(
("tests/data/files/test_R1_L001.fq.gz",
"tests/data/files/test_R2_L001.fq.gz"),
FakeSamplesResource(),
)
assert b3.call_count == 2
def test_api_failures(caplog):
with patch("boto3.session.Session"):
files = ("tests/data/files/test_R1_L001.fq.gz",
"tests/data/files/test_R2_L001.fq.gz")
with pytest.raises(UploadException) as e:
upload_sequence(files, FakeSamplesResource("init_multipart"))
assert "Could not initialize upload" in str(e.value)
def test_callback_retry_handling():
with patch("boto3.session.Session") as _:
sample_resource = FakeSamplesResource()
files = ("tests/data/files/test_R1_L001.fq.gz",
"tests/data/files/test_R2_L001.fq.gz")
upload_sequence(files, sample_resource)
assert (sample_resource._client.session.adapters[
"http://localhost:3000/s3_confirm"].max_retries.total == 3)
assert (sample_resource._client.session.adapters[
"http://localhost:3000/s3_confirm"].max_retries.method_whitelist is
False)
def test_unicode_filenames(caplog):
file_list = [
(u"tests/data/files/François.fq", "Francois.fq"),
(u"tests/data/files/Málaga.fasta", "Malaga.fasta"),
(u"tests/data/files/Röö.fastq", "Roo.fastq"),
]
# should raise if --coerce-ascii not passed
for before, after in file_list:
with pytest.raises(OneCodexException) as e:
upload_sequence(before, FakeSession(), FakeSamplesResource())
assert "must be ascii" in str(e.value)
# make sure log gets warnings when we rename files
for before, after in file_list:
upload_sequence(before, FakeSession(), FakeSamplesResource(), coerce_ascii=True)
for _, after in file_list:
assert after in caplog.text
def test_unicode_filenames(caplog):
with patch("boto3.session.Session"):
file_list = [
(u"tests/data/files/François.fq", "Francois.fq"),
(u"tests/data/files/Málaga.fasta", "Malaga.fasta"),
(u"tests/data/files/Röö.fastq", "Roo.fastq"),
]
# should raise if --coerce-ascii not passed
for before, after in file_list:
with pytest.raises(OneCodexException) as e:
upload_sequence(before, FakeSamplesResource())
assert "must be ascii" in str(e.value)
# make sure log gets warnings when we rename files
for before, _ in file_list:
upload_sequence(before, FakeSamplesResource(), coerce_ascii=True)
for _, after in file_list:
assert after in caplog.text
def test_api_failures(caplog):
files = ("tests/data/files/test_R1_L001.fq.gz",
"tests/data/files/test_R2_L001.fq.gz")
with pytest.raises(UploadException) as e:
upload_sequence(files, FakeProxySession(200),
FakeSamplesResource("init"))
assert "Could not initialize upload" in str(e.value)
# Test direct upload not via the proxy
with pytest.raises(UploadException) as e:
with patch("boto3.client") as b3:
upload_sequence(files, FakeProxySession(200),
FakeSamplesResource("confirm", via_proxy=False))
assert b3.call_count == 1
assert "Callback could not be completed" in str(e.value)
# Test 400 on proxy
with pytest.raises(UploadException) as e:
upload_sequence(files, FakeProxySession(400, no_msg=True),
FakeSamplesResource())
assert "File could not be uploaded" in str(e.value)
# Test 500 on proxy
# Test connectivity
with pytest.raises(UploadException) as e:
raise_connectivity_error("filename")
assert "experiencing connectivity" in str(e.value)
def test_api_failures(caplog):
files = ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz")
with pytest.raises(UploadException) as e:
upload_sequence(files, FakeSession(), FakeSamplesResource("init"))
assert "Could not initialize upload" in str(e.value)
# Test direct upload not via the proxy
with pytest.raises(UploadException) as e:
with patch("boto3.client") as b3:
upload_sequence(files, FakeSession(), FakeSamplesResource("confirm", via_proxy=False))
assert b3.call_count == 1
assert "Callback could not be completed" in str(e.value)
# Test 400 on proxy
with pytest.raises(UploadException) as e:
upload_sequence(files, FakeSessionProxyFails(400, no_msg=True), FakeSamplesResource())
assert "File could not be uploaded" in str(e.value)
# Test 500 on proxy
# Test connectivity
with pytest.raises(UploadException) as e:
raise_connectivity_error("filename")
assert "experiencing connectivity" in str(e.value)
def test_single_end_files():
upload_sequence("tests/data/files/test_R1_L001.fq.gz",
FakeProxySession(200), FakeSamplesResource())
def upload(
cls,
files,
metadata=None,
tags=None,
project=None,
coerce_ascii=False,
progressbar=None,
sample_id=None,
external_sample_id=None,
):
"""Upload a series of files to the One Codex server.
Parameters
----------
files : `string` or `tuple`
A single path to a file on the system, or a tuple containing a pairs of paths. Tuple
values will be interleaved as paired-end reads and both files should contain the same
number of records. Paths to single files will be uploaded as-is.
metadata : `dict`, optional
tags : `list`, optional
A list of optional tags to create. Tags must be passed as dictionaries with a single key
`name` and the tag name, e.g., {"name": "my tag"}. New tags will be created on-the-fly
as needed.
project : `string`, optional
UUID of project to associate this sample with.
coerce_ascii : `bool`, optional
If true, rename unicode filenames to ASCII and issue warning.
progressbar : `click.progressbar`, optional
If passed, display a progress bar using Click.
sample_id : `string`, optional
If passed, will upload the file(s) to the sample with that id. Only works if the sample was pre-uploaded
external_sample_id : `string`, optional
If passed, will upload the file(s) to the sample with that metadata external id. Only works if the sample was pre-uploaded
Returns
-------
A `Samples` object upon successful upload. None if the upload failed.
"""
if not isinstance(files, string_types) and not isinstance(
files, tuple):
raise OneCodexException(
"Please pass a string or tuple or forward and reverse filepaths."
)
if sample_id and external_sample_id:
raise OneCodexException(
"Only pass sample_id OR external_sample_id, not both.")
project = get_project(project)
sample_id = upload_sequence(
files,
cls._resource,
metadata=metadata,
tags=tags,
project=project,
coerce_ascii=coerce_ascii,
progressbar=progressbar,
sample_id=sample_id,
external_sample_id=external_sample_id,
)
return cls.get(sample_id)
def test_single_end_files():
with patch("boto3.session.Session") as b3:
upload_sequence("tests/data/files/test_R1_L001.fq.gz",
FakeSamplesResource())
assert b3.call_count == 1
def test_paired_end_files():
upload_sequence(
("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz"),
FakeSession(),
FakeSamplesResource(),
)
def test_single_end_files():
upload_sequence("tests/data/files/test_R1_L001.fq.gz", FakeSession(), FakeSamplesResource())