def test_paired_end_files(): upload_sequence( ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz"), FakeProxySession(200), FakeSamplesResource(), )
def test_upload_lots_of_files(files, n_uploads, fxi_calls, fxp_calls, size_calls): fake_size = lambda filename: int(float(filename.split(".")[1])) # noqa uso = "onecodex.lib.upload.upload_sequence_fileobj" udo = "onecodex.lib.upload.upload_document_fileobj" fxi = "onecodex.lib.upload.FASTXInterleave" fxp = "onecodex.lib.upload.FilePassthru" sz = "onecodex.lib.upload.os.path.getsize" with patch(uso) as upload_sequence_fileobj, patch(fxi) as interleave, patch(fxp) as passthru: with patch(sz, size_effect=fake_size) as size: upload_sequence(files, FakeSession(), FakeSamplesResource()) assert upload_sequence_fileobj.call_count == n_uploads assert interleave.call_count == fxi_calls assert passthru.call_count == fxp_calls assert size.call_count == size_calls with patch(udo) as upload_document_fileobj, patch(fxp) as passthru: with patch(sz, size_effect=fake_size) as size: files = files[0] if isinstance(files, tuple) else files upload_document(files, FakeSession(), FakeDocumentsResource()) assert upload_document_fileobj.call_count == n_uploads assert passthru.call_count == fxp_calls + fxi_calls assert size.call_count == fxp_calls + fxi_calls
def test_upload_lots_of_files(files, n_uploads, fxi_calls, fxp_calls, size_calls): fake_size = lambda filename: int(float(filename.split(".")[1])) # noqa uso = "onecodex.lib.upload.upload_sequence_fileobj" udo = "onecodex.lib.upload.upload_document_fileobj" fxi = "onecodex.lib.upload.FASTXInterleave" fxp = "onecodex.lib.upload.FilePassthru" sz = "onecodex.lib.upload.os.path.getsize" with patch(uso) as upload_sequence_fileobj, patch( fxi) as interleave, patch(fxp) as passthru: with patch(sz, size_effect=fake_size) as size: upload_sequence(files, FakeProxySession(200), FakeSamplesResource()) assert upload_sequence_fileobj.call_count == n_uploads assert interleave.call_count == fxi_calls assert passthru.call_count == fxp_calls assert size.call_count == size_calls with patch(udo) as upload_document_fileobj, patch(fxp) as passthru: with patch(sz, size_effect=fake_size) as size: files = files[0] if isinstance(files, tuple) else files upload_document(files, FakeDocumentsResource()) assert upload_document_fileobj.call_count == n_uploads assert passthru.call_count == fxp_calls + fxi_calls assert size.call_count == fxp_calls + fxi_calls
def test_upload_lots_of_files(files, n_uploads, fxi_calls, fxp_calls): with patch("boto3.session.Session"): fake_size = lambda filename: int(float(filename.split(".")[1])) # noqa uso = "onecodex.lib.upload._upload_sequence_fileobj" udo = "onecodex.lib.upload._upload_document_fileobj" fxi = "onecodex.lib.files.PairedEndFiles" fxp = "onecodex.lib.files.FilePassthru" sz = "os.path.getsize" sample_resource = FakeSamplesResource() with patch(uso) as upload_sequence_fileobj, patch( fxi) as paired, patch(fxp) as passthru: with patch(sz, size_effect=fake_size): upload_sequence(files, sample_resource) assert upload_sequence_fileobj.call_count == n_uploads assert paired.call_count == fxi_calls assert passthru.call_count == fxp_calls # Need to patch from `upload.` instead of `files.` for that test with patch(udo) as upload_document_fileobj, patch( "onecodex.lib.upload.FilePassthru") as passthru: with patch(sz, size_effect=fake_size): files = files[0] if isinstance(files, tuple) else files upload_document(files, FakeDocumentsResource()) assert (upload_document_fileobj.call_count == n_uploads - 1 if isinstance(files, tuple) else n_uploads) assert passthru.call_count == fxp_calls + fxi_calls
def test_paired_end_files(): with patch("boto3.session.Session") as b3: upload_sequence( ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz"), FakeSamplesResource(), ) assert b3.call_count == 2
def test_api_failures(caplog): with patch("boto3.session.Session"): files = ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz") with pytest.raises(UploadException) as e: upload_sequence(files, FakeSamplesResource("init_multipart")) assert "Could not initialize upload" in str(e.value)
def test_callback_retry_handling(): with patch("boto3.session.Session") as _: sample_resource = FakeSamplesResource() files = ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz") upload_sequence(files, sample_resource) assert (sample_resource._client.session.adapters[ "http://localhost:3000/s3_confirm"].max_retries.total == 3) assert (sample_resource._client.session.adapters[ "http://localhost:3000/s3_confirm"].max_retries.method_whitelist is False)
def test_unicode_filenames(caplog): file_list = [ (u"tests/data/files/François.fq", "Francois.fq"), (u"tests/data/files/Málaga.fasta", "Malaga.fasta"), (u"tests/data/files/Röö.fastq", "Roo.fastq"), ] # should raise if --coerce-ascii not passed for before, after in file_list: with pytest.raises(OneCodexException) as e: upload_sequence(before, FakeSession(), FakeSamplesResource()) assert "must be ascii" in str(e.value) # make sure log gets warnings when we rename files for before, after in file_list: upload_sequence(before, FakeSession(), FakeSamplesResource(), coerce_ascii=True) for _, after in file_list: assert after in caplog.text
def test_unicode_filenames(caplog): with patch("boto3.session.Session"): file_list = [ (u"tests/data/files/François.fq", "Francois.fq"), (u"tests/data/files/Málaga.fasta", "Malaga.fasta"), (u"tests/data/files/Röö.fastq", "Roo.fastq"), ] # should raise if --coerce-ascii not passed for before, after in file_list: with pytest.raises(OneCodexException) as e: upload_sequence(before, FakeSamplesResource()) assert "must be ascii" in str(e.value) # make sure log gets warnings when we rename files for before, _ in file_list: upload_sequence(before, FakeSamplesResource(), coerce_ascii=True) for _, after in file_list: assert after in caplog.text
def test_api_failures(caplog): files = ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz") with pytest.raises(UploadException) as e: upload_sequence(files, FakeProxySession(200), FakeSamplesResource("init")) assert "Could not initialize upload" in str(e.value) # Test direct upload not via the proxy with pytest.raises(UploadException) as e: with patch("boto3.client") as b3: upload_sequence(files, FakeProxySession(200), FakeSamplesResource("confirm", via_proxy=False)) assert b3.call_count == 1 assert "Callback could not be completed" in str(e.value) # Test 400 on proxy with pytest.raises(UploadException) as e: upload_sequence(files, FakeProxySession(400, no_msg=True), FakeSamplesResource()) assert "File could not be uploaded" in str(e.value) # Test 500 on proxy # Test connectivity with pytest.raises(UploadException) as e: raise_connectivity_error("filename") assert "experiencing connectivity" in str(e.value)
def test_api_failures(caplog): files = ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz") with pytest.raises(UploadException) as e: upload_sequence(files, FakeSession(), FakeSamplesResource("init")) assert "Could not initialize upload" in str(e.value) # Test direct upload not via the proxy with pytest.raises(UploadException) as e: with patch("boto3.client") as b3: upload_sequence(files, FakeSession(), FakeSamplesResource("confirm", via_proxy=False)) assert b3.call_count == 1 assert "Callback could not be completed" in str(e.value) # Test 400 on proxy with pytest.raises(UploadException) as e: upload_sequence(files, FakeSessionProxyFails(400, no_msg=True), FakeSamplesResource()) assert "File could not be uploaded" in str(e.value) # Test 500 on proxy # Test connectivity with pytest.raises(UploadException) as e: raise_connectivity_error("filename") assert "experiencing connectivity" in str(e.value)
def test_single_end_files(): upload_sequence("tests/data/files/test_R1_L001.fq.gz", FakeProxySession(200), FakeSamplesResource())
def upload( cls, files, metadata=None, tags=None, project=None, coerce_ascii=False, progressbar=None, sample_id=None, external_sample_id=None, ): """Upload a series of files to the One Codex server. Parameters ---------- files : `string` or `tuple` A single path to a file on the system, or a tuple containing a pairs of paths. Tuple values will be interleaved as paired-end reads and both files should contain the same number of records. Paths to single files will be uploaded as-is. metadata : `dict`, optional tags : `list`, optional A list of optional tags to create. Tags must be passed as dictionaries with a single key `name` and the tag name, e.g., {"name": "my tag"}. New tags will be created on-the-fly as needed. project : `string`, optional UUID of project to associate this sample with. coerce_ascii : `bool`, optional If true, rename unicode filenames to ASCII and issue warning. progressbar : `click.progressbar`, optional If passed, display a progress bar using Click. sample_id : `string`, optional If passed, will upload the file(s) to the sample with that id. Only works if the sample was pre-uploaded external_sample_id : `string`, optional If passed, will upload the file(s) to the sample with that metadata external id. Only works if the sample was pre-uploaded Returns ------- A `Samples` object upon successful upload. None if the upload failed. """ if not isinstance(files, string_types) and not isinstance( files, tuple): raise OneCodexException( "Please pass a string or tuple or forward and reverse filepaths." ) if sample_id and external_sample_id: raise OneCodexException( "Only pass sample_id OR external_sample_id, not both.") project = get_project(project) sample_id = upload_sequence( files, cls._resource, metadata=metadata, tags=tags, project=project, coerce_ascii=coerce_ascii, progressbar=progressbar, sample_id=sample_id, external_sample_id=external_sample_id, ) return cls.get(sample_id)
def test_single_end_files(): with patch("boto3.session.Session") as b3: upload_sequence("tests/data/files/test_R1_L001.fq.gz", FakeSamplesResource()) assert b3.call_count == 1
def test_paired_end_files(): upload_sequence( ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz"), FakeSession(), FakeSamplesResource(), )
def test_single_end_files(): upload_sequence("tests/data/files/test_R1_L001.fq.gz", FakeSession(), FakeSamplesResource())