def test_add_reference_unknown_handler(runner):
with runner.isolated_filesystem():
artifact = artifacts.Artifact(type='dataset', name='my-arty')
artifact.add_reference('http://example.com/somefile.txt', name='ref')
assert artifact.digest == '5b8876252f3ca922c164de380089c9ae'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['ref'] == {
'digest': 'http://example.com/somefile.txt',
'ref': 'http://example.com/somefile.txt'
}
def test_add_reference_local_file_no_checksum(runner):
with runner.isolated_filesystem():
open('file1.txt', 'w').write('hello')
artifact = artifacts.Artifact(type='dataset', name='my-arty')
artifact.add_reference('file://file1.txt', checksum=False)
assert artifact.digest == '2f66dd01e5aea4af52445f7602fe88a0'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['file1.txt'] == {
'digest': 'file://file1.txt',
'ref': 'file://file1.txt'
}
def test_add_dir(runner):
with runner.isolated_filesystem():
open('file1.txt', 'w').write('hello')
artifact = artifacts.Artifact(type='dataset', name='my-arty')
artifact.add_dir('.')
assert artifact.digest == 'a00c2239f036fb656c1dcbf9a32d89b4'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['file1.txt'] == {
'digest': 'XUFAKrxLKna5cZ2REBfFkg==',
'size': 5
}
def test_add_reference_named_local_file(runner):
with runner.isolated_filesystem():
open('file1.txt', 'w').write('hello')
artifact = artifacts.Artifact(type='dataset', name='my-arty')
artifact.add_reference('file://file1.txt', name='great-file.txt')
assert artifact.digest == '585b9ada17797e37c9cbab391e69b8c5'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['great-file.txt'] == {
'digest': 'XUFAKrxLKna5cZ2REBfFkg==',
'ref': 'file://file1.txt',
'size': 5
}
def test_add_reference_s3_no_checksum(runner):
with runner.isolated_filesystem():
open('file1.txt', 'w').write('hello')
artifact = artifacts.Artifact(type='dataset', name='my-arty')
# TODO: Should we require name in this case?
artifact.add_reference('s3://my_bucket/file1.txt', checksum=False)
assert artifact.digest == '52631787ed3579325f985dc0f2374040'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['file1.txt'] == {
'digest': 's3://my_bucket/file1.txt',
'ref': 's3://my_bucket/file1.txt'
}
def test_add_named_dir(runner):
with runner.isolated_filesystem():
open('file1.txt', 'w').write('hello')
artifact = artifacts.Artifact(type='dataset', name='my-arty')
artifact.add_dir('.', name='subdir')
assert artifact.digest == 'a757208d042e8627b2970d72a71bed5b'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['subdir/file1.txt'] == {
'digest': 'XUFAKrxLKna5cZ2REBfFkg==',
'size': 5
}
def test_add_gs_reference_object(runner, mocker):
with runner.isolated_filesystem():
artifact = artifacts.Artifact(type="dataset", name='my-arty')
mock_gcs(artifact)
artifact.add_reference("gs://my-bucket/my_object.pb")
assert artifact.digest == '8aec0d6978da8c2b0bf5662b3fd043a4'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['my_object.pb'] == {
'digest': '1234567890abcde',
'ref': 'gs://my-bucket/my_object.pb',
'extra': {
'etag': '1234567890abcde',
'versionID': '1'
},
'size': 10
}
def test_add_gs_reference_object_with_name(runner, mocker):
with runner.isolated_filesystem():
artifact = artifacts.Artifact(type="dataset", name='my-arty')
mock_gcs(artifact)
artifact.add_reference("gs://my-bucket/my_object.pb",
name="renamed.pb")
assert artifact.digest == 'bd85fe009dc9e408a5ed9b55c95f47b2'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['renamed.pb'] == {
'digest': '1234567890abcde',
'ref': 'gs://my-bucket/my_object.pb',
'extra': {
'etag': '1234567890abcde',
'versionID': '1'
},
'size': 10
}
def test_add_gs_reference_path(runner, mocker, capsys):
with runner.isolated_filesystem():
artifact = artifacts.Artifact(type="dataset", name='my-arty')
mock_gcs(artifact, path=True)
artifact.add_reference("gs://my-bucket/")
assert artifact.digest == '17955d00a20e1074c3bc96c74b724bfe'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['my_object.pb'] == {
'digest': '1234567890abcde',
'ref': 'gs://my-bucket/my_object.pb',
'extra': {
'etag': '1234567890abcde',
'versionID': '1'
},
'size': 10
}
_, err = capsys.readouterr()
assert "Generating checksum" in err
def test_add_reference_local_dir(runner):
with runner.isolated_filesystem():
open('file1.txt', 'w').write('hello')
open('file2.txt', 'w').write('dude')
artifact = artifacts.Artifact(type='dataset', name='my-arty')
artifact.add_reference('file://' + os.getcwd())
assert artifact.digest == '5e8e98ebd59cc93b58d0cb26432d4720'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['file1.txt'] == {
'digest': 'XUFAKrxLKna5cZ2REBfFkg==',
'ref': 'file://' + os.getcwd() + '/file1.txt',
'size': 5
}
assert manifest['contents']['file2.txt'] == {
'digest': 'E7c+2uhEOZC+GqjxpIO8Jw==',
'ref': 'file://' + os.getcwd() + '/file2.txt',
'size': 4
}
def test_add_http_reference_path(runner):
with runner.isolated_filesystem():
artifact = artifacts.Artifact(type='dataset', name='my-arty')
mock_http(artifact,
headers={
'ETag': '"abc"',
'Content-Length': "256",
})
artifact.add_reference("http://example.com/file1.txt")
assert artifact.digest == '48237ccc050a88af9dcd869dd5a7e9f4'
manifest = artifact.manifest.to_manifest_json()
assert manifest['contents']['file1.txt'] == {
'digest': 'abc',
'ref': 'http://example.com/file1.txt',
'size': 256,
'extra': {
'etag': '"abc"',
},
}
def test_add_s3_max_objects(runner, mocker, capsys):
with runner.isolated_filesystem():
artifact = artifacts.Artifact(type="dataset", name='my-arty')
mock_boto(artifact, path=True)
with pytest.raises(ValueError):
artifact.add_reference("s3://my-bucket/", max_objects=1)
