def test_brackets():
pkg = Package()
pkg.set('asdf/jkl', LOCAL_MANIFEST)
pkg.set('asdf/qwer', LOCAL_MANIFEST)
pkg.set('qwer/asdf', LOCAL_MANIFEST)
assert set(pkg.keys()) == {'asdf', 'qwer'}
pkg2 = pkg['asdf']
assert set(pkg2.keys()) == {'jkl', 'qwer'}
assert pkg['asdf']['qwer'].get() == pathlib.Path(LOCAL_MANIFEST).as_uri()
assert pkg['asdf']['qwer'] == pkg['asdf/qwer'] == pkg[('asdf', 'qwer')]
assert pkg[[]] == pkg
pkg = (Package().set(
'foo', os.path.join(os.path.dirname(__file__), 'data', 'foo.txt'),
{'foo': 'blah'}))
pkg['foo'].meta['target'] = 'unicode'
pkg.build()
assert pkg['foo'].deserialize() == '123\n'
assert pkg['foo']() == '123\n'
with pytest.raises(KeyError):
pkg['baz']
with pytest.raises(TypeError):
pkg[b'asdf']
with pytest.raises(TypeError):
pkg[0]
def test_build(tmpdir):
"""Verify that build dumps the manifest to appdirs directory."""
new_pkg = Package()
# Create a dummy file to add to the package.
test_file_name = 'bar'
with open(test_file_name, "w") as fd:
fd.write('test_file_content_string')
test_file = Path(fd.name)
# Build a new package into the local registry.
new_pkg = new_pkg.set('foo', test_file_name)
top_hash = new_pkg.build("Quilt/Test")
# Verify manifest is registered by hash.
out_path = Path(BASE_PATH, ".quilt/packages", top_hash)
with open(out_path) as fd:
pkg = Package.load(fd)
assert test_file.resolve().as_uri() == pkg['foo'].physical_keys[0]
# Verify latest points to the new location.
named_pointer_path = Path(BASE_PATH,
".quilt/named_packages/Quilt/Test/latest")
with open(named_pointer_path) as fd:
assert fd.read().replace('\n', '') == top_hash
# Test unnamed packages.
new_pkg = Package()
new_pkg = new_pkg.set('bar', test_file_name)
top_hash = new_pkg.build()
out_path = Path(BASE_PATH, ".quilt/packages", top_hash)
with open(out_path) as fd:
pkg = Package.load(fd)
assert test_file.resolve().as_uri() == pkg['bar'].physical_keys[0]
def test_brackets(self):
pkg = Package()
pkg.set('asdf/jkl', LOCAL_MANIFEST)
pkg.set('asdf/qwer', LOCAL_MANIFEST)
pkg.set('qwer/asdf', LOCAL_MANIFEST)
assert set(pkg.keys()) == {'asdf', 'qwer'}
pkg2 = pkg['asdf']
assert set(pkg2.keys()) == {'jkl', 'qwer'}
assert pkg['asdf']['qwer'].get() == LOCAL_MANIFEST.as_uri()
assert pkg['asdf']['qwer'] == pkg['asdf/qwer'] == pkg[('asdf', 'qwer')]
assert pkg[[]] == pkg
pkg = (Package().set('foo', DATA_DIR / 'foo.txt', {'foo': 'blah'}))
pkg['foo'].meta['target'] = 'unicode'
pkg.build()
assert pkg['foo'].deserialize() == '123\n'
assert pkg['foo']() == '123\n'
with pytest.raises(KeyError):
pkg['baz']
with pytest.raises(TypeError):
pkg[b'asdf']
with pytest.raises(TypeError):
pkg[0]
def test_manifest(self):
pkg = Package()
pkg.set('as/df', LOCAL_MANIFEST)
pkg.set('as/qw', LOCAL_MANIFEST)
top_hash = pkg.build().top_hash
manifest = list(pkg.manifest)
pkg2 = Package.browse(top_hash=top_hash, registry='local')
assert list(pkg.manifest) == list(pkg2.manifest)
def test_manifest():
pkg = Package()
pkg.set('as/df', LOCAL_MANIFEST)
pkg.set('as/qw', LOCAL_MANIFEST)
top_hash = pkg.build()
manifest = list(pkg.manifest)
pkg2 = Package.browse(pkg_hash=top_hash)
assert list(pkg.manifest) == list(pkg2.manifest)
def test_map():
pkg = Package()
pkg.set('as/df', LOCAL_MANIFEST)
pkg.set('as/qw', LOCAL_MANIFEST)
assert set(pkg.map(lambda lk, entry: lk)) == {'as/df', 'as/qw'}
pkg['as'].set_meta({'foo': 'bar'})
assert set(pkg.map(lambda lk, entry: lk, include_directories=True)) ==\
{'as/df', 'as/qw', 'as/'}
def test_iter():
pkg = Package()
assert not pkg
pkg.set('asdf', LOCAL_MANIFEST)
assert list(pkg) == ['asdf']
pkg.set('jkl;', REMOTE_MANIFEST)
assert set(pkg) == {'asdf', 'jkl;'}
def test_keys():
pkg = Package()
assert not pkg.keys()
pkg.set('asdf', LOCAL_MANIFEST)
assert set(pkg.keys()) == {'asdf'}
pkg.set('jkl;', REMOTE_MANIFEST)
assert set(pkg.keys()) == {'asdf', 'jkl;'}
pkg.delete('asdf')
assert set(pkg.keys()) == {'jkl;'}
def test_default_registry(self):
new_pkg = Package()
# Create a dummy file to add to the package.
test_file_name = 'bar'
with open(test_file_name, "w") as fd:
fd.write('test_file_content_string')
test_file = Path(fd.name)
# Build a new package into the local registry.
new_pkg = new_pkg.set('foo', test_file_name)
top_hash = new_pkg.build("Quilt/Test").top_hash
# Verify manifest is registered by hash.
out_path = Path(BASE_PATH, ".quilt/packages", top_hash)
with open(out_path) as fd:
pkg = Package.load(fd)
assert test_file.resolve().as_uri() == pkg['foo'].physical_keys[0]
# Verify latest points to the new location.
named_pointer_path = Path(BASE_PATH,
".quilt/named_packages/Quilt/Test/latest")
with open(named_pointer_path) as fd:
assert fd.read().replace('\n', '') == top_hash
# Test unnamed packages.
new_pkg = Package()
new_pkg = new_pkg.set('bar', test_file_name)
top_hash = new_pkg.build().top_hash
out_path = Path(BASE_PATH, ".quilt/packages", top_hash)
with open(out_path) as fd:
pkg = Package.load(fd)
assert test_file.resolve().as_uri() == pkg['bar'].physical_keys[0]
new_base_path = Path(BASE_PATH, ".quilttest")
with patch('t4.packages.get_from_config') as mock_config:
mock_config.return_value = new_base_path
top_hash = new_pkg.build("Quilt/Test").top_hash
out_path = Path(new_base_path, ".quilt/packages",
top_hash).resolve()
with open(out_path) as fd:
pkg = Package.load(fd)
assert test_file.resolve().as_uri(
) == pkg['bar'].physical_keys[0]
with patch('t4.packages.get_from_config') as mock_config:
mock_config.return_value = new_base_path
new_pkg.push("Quilt/Test")
with open(out_path) as fd:
pkg = Package.load(fd)
assert pkg['bar'].physical_keys[0].endswith(
'.quilttest/Quilt/Test/bar')
def test_long_repr():
pkg = Package()
for i in range(30):
pkg.set('path{}/asdf'.format(i), LOCAL_MANIFEST)
r = repr(pkg)
assert r.count('\n') == 20
assert r[-4:] == '...\n'
pkg = Package()
for i in range(10):
pkg.set('path{}/asdf'.format(i), LOCAL_MANIFEST)
pkg.set('path{}/qwer'.format(i), LOCAL_MANIFEST)
pkgrepr = repr(pkg)
assert pkgrepr.count('\n') == 20
assert pkgrepr.find('path9/') > 0
def test_filter():
pkg = Package()
pkg.set('as/df', LOCAL_MANIFEST)
pkg.set('as/qw', LOCAL_MANIFEST)
assert list(pkg.filter(lambda lk, entry: lk == 'as/df')) == [
('as/df', pkg['as/df'])
]
pkg['as'].set_meta({'foo': 'bar'})
assert list(pkg.filter(lambda lk, entry: lk == 'as/df')) == [
('as/df', pkg['as/df'])
]
assert list(
pkg.filter(lambda lk, entry: lk == 'as/',
include_directories=True)) == [('as/', pkg['as'])]
def test_local_push(tmpdir):
""" Verify loading local manifest and data into S3. """
with patch('t4.packages.put_bytes') as bytes_mock, \
patch('t4.data_transfer._copy_local_file') as file_mock, \
patch('t4.packages.get_remote_registry') as config_mock:
config_mock.return_value = tmpdir / 'package_contents'
new_pkg = Package()
contents = 'blah'
test_file = pathlib.Path(tmpdir) / 'bar'
test_file.write_text(contents)
new_pkg = new_pkg.set('foo', test_file)
new_pkg.push('Quilt/package', tmpdir / 'package_contents')
push_uri = pathlib.Path(tmpdir, 'package_contents').as_uri()
# Manifest copied
top_hash = new_pkg.top_hash()
bytes_mock.assert_any_call(
top_hash.encode(),
push_uri + '/.quilt/named_packages/Quilt/package/latest')
bytes_mock.assert_any_call(ANY,
push_uri + '/.quilt/packages/' + top_hash)
# Data copied
file_mock.assert_called_once_with(
ANY, len(contents), str(test_file),
str(tmpdir / 'package_contents/Quilt/package/foo'), {})
def test_load_into_t4(tmpdir):
""" Verify loading local manifest and data into S3. """
with patch('t4.packages.put_bytes') as bytes_mock, \
patch('t4.data_transfer._upload_file') as file_mock, \
patch('t4.packages.get_remote_registry') as config_mock:
config_mock.return_value = 's3://my_test_bucket'
new_pkg = Package()
# Create a dummy file to add to the package.
contents = 'blah'
test_file = pathlib.Path(tmpdir) / 'bar'
test_file.write_text(contents)
new_pkg = new_pkg.set('foo', test_file)
new_pkg.push('Quilt/package', 's3://my_test_bucket/')
# Manifest copied
top_hash = new_pkg.top_hash()
bytes_mock.assert_any_call(
top_hash.encode(),
's3://my_test_bucket/.quilt/named_packages/Quilt/package/latest')
bytes_mock.assert_any_call(
ANY, 's3://my_test_bucket/.quilt/packages/' + top_hash)
# Data copied
file_mock.assert_called_once_with(ANY, len(contents), str(test_file),
'my_test_bucket',
'Quilt/package/foo', {})
def test_load_into_t4(self):
""" Verify loading local manifest and data into S3. """
top_hash = '5333a204bbc6e21607c2bc842f4a77d2e21aa6147cf2bf493dbf6282188d01ca'
self.s3_stubber.add_response(method='put_object',
service_response={'VersionId': 'v1'},
expected_params={
'Body': ANY,
'Bucket': 'my_test_bucket',
'Key': 'Quilt/package/foo',
'Metadata': {
'helium': '{}'
}
})
self.s3_stubber.add_response(method='put_object',
service_response={'VersionId': 'v2'},
expected_params={
'Body': ANY,
'Bucket': 'my_test_bucket',
'Key': '.quilt/packages/' + top_hash,
'Metadata': {
'helium': 'null'
}
})
self.s3_stubber.add_response(
method='put_object',
service_response={'VersionId': 'v3'},
expected_params={
'Body': top_hash.encode(),
'Bucket': 'my_test_bucket',
'Key': '.quilt/named_packages/Quilt/package/1234567890',
'Metadata': {
'helium': 'null'
}
})
self.s3_stubber.add_response(
method='put_object',
service_response={'VersionId': 'v4'},
expected_params={
'Body': top_hash.encode(),
'Bucket': 'my_test_bucket',
'Key': '.quilt/named_packages/Quilt/package/latest',
'Metadata': {
'helium': 'null'
}
})
new_pkg = Package()
# Create a dummy file to add to the package.
contents = 'blah'
test_file = Path('bar')
test_file.write_text(contents)
new_pkg = new_pkg.set('foo', test_file)
with patch('time.time', return_value=1234567890):
new_pkg.push('Quilt/package', 's3://my_test_bucket/')
def test_reduce():
pkg = Package()
pkg.set('as/df', LOCAL_MANIFEST)
pkg.set('as/qw', LOCAL_MANIFEST)
assert pkg.reduce(lambda a, b: a) == ('as/df', pkg['as/df'])
assert pkg.reduce(lambda a, b: b) == ('as/qw', pkg['as/qw'])
assert list(pkg.reduce(lambda a, b: a + [b],
[])) == [('as/df', pkg['as/df']),
('as/qw', pkg['as/qw'])]
pkg['as'].set_meta({'foo': 'bar'})
assert pkg.reduce(lambda a, b: b, include_directories=True) ==\
('as/qw', pkg['as/qw'])
assert list(pkg.reduce(lambda a, b: a + [b], [],
include_directories=True)) == [
('as/', pkg['as']), ('as/df', pkg['as/df']),
('as/qw', pkg['as/qw'])
]
def test_tophash_changes(tmpdir):
test_file = tmpdir / 'test.txt'
test_file.write_text('asdf', 'utf-8')
pkg = Package()
th1 = pkg.top_hash()
pkg.set('asdf', test_file)
pkg.build()
th2 = pkg.top_hash()
assert th1 != th2
test_file.write_text('jkl', 'utf-8')
pkg.set('jkl', test_file)
pkg.build()
th3 = pkg.top_hash()
assert th1 != th3
assert th2 != th3
pkg.delete('jkl')
th4 = pkg.top_hash()
assert th2 == th4
def test_dir_meta(tmpdir):
test_meta = {'test': 'meta'}
pkg = Package()
pkg.set('asdf/jkl', LOCAL_MANIFEST)
pkg.set('asdf/qwer', LOCAL_MANIFEST)
pkg.set('qwer/asdf', LOCAL_MANIFEST)
pkg.set('qwer/as/df', LOCAL_MANIFEST)
pkg.build()
assert pkg['asdf'].get_meta() == {}
assert pkg.get_meta() == {}
assert pkg['qwer']['as'].get_meta() == {}
pkg['asdf'].set_meta(test_meta)
assert pkg['asdf'].get_meta() == test_meta
pkg['qwer']['as'].set_meta(test_meta)
assert pkg['qwer']['as'].get_meta() == test_meta
pkg.set_meta(test_meta)
assert pkg.get_meta() == test_meta
dump_path = os.path.join(tmpdir, 'test_meta')
with open(dump_path, 'w') as f:
pkg.dump(f)
with open(dump_path) as f:
pkg2 = Package.load(f)
assert pkg2['asdf'].get_meta() == test_meta
assert pkg2['qwer']['as'].get_meta() == test_meta
assert pkg2.get_meta() == test_meta
def test_diff():
new_pkg = Package()
# Create a dummy file to add to the package.
test_file_name = 'bar'
with open(test_file_name, "w") as fd:
fd.write('test_file_content_string')
test_file = Path(fd.name)
# Build a new package into the local registry.
new_pkg = new_pkg.set('foo', test_file_name)
top_hash = new_pkg.build("Quilt/Test")
p1 = Package.browse('Quilt/Test')
p2 = Package.browse('Quilt/Test')
assert p1.diff(p2) == ([], [], [])
def test_local_push(self):
""" Verify loading local manifest and data into a local dir. """
top_hash = '5333a204bbc6e21607c2bc842f4a77d2e21aa6147cf2bf493dbf6282188d01ca'
new_pkg = Package()
contents = 'blah'
test_file = Path('bar')
test_file.write_text(contents)
new_pkg = new_pkg.set('foo', test_file)
new_pkg.push('Quilt/package', 'package_contents')
push_dir = Path('package_contents')
assert (push_dir / '.quilt/named_packages/Quilt/package/latest'
).read_text() == top_hash
assert (push_dir / ('.quilt/packages/' + top_hash)).exists()
assert (push_dir / 'Quilt/package/foo').read_text() == contents
def test_repr():
TEST_REPR = ("asdf\n"
"path1/\n"
" asdf\n"
" qwer\n"
"path2/\n"
" first/\n"
" asdf\n"
" second/\n"
" asdf\n"
"qwer\n")
pkg = Package()
pkg.set('asdf', LOCAL_MANIFEST)
pkg.set('qwer', LOCAL_MANIFEST)
pkg.set('path1/asdf', LOCAL_MANIFEST)
pkg.set('path1/qwer', LOCAL_MANIFEST)
pkg.set('path2/first/asdf', LOCAL_MANIFEST)
pkg.set('path2/second/asdf', LOCAL_MANIFEST)
assert repr(pkg) == TEST_REPR
def test_local_repr(self):
TEST_REPR = ("(local Package)\n"
" └─asdf\n"
" └─path1/\n"
" └─asdf\n"
" └─qwer\n"
" └─path2/\n"
" └─first/\n"
" └─asdf\n"
" └─second/\n"
" └─asdf\n"
" └─qwer\n")
pkg = Package()
pkg.set('asdf', LOCAL_MANIFEST)
pkg.set('qwer', LOCAL_MANIFEST)
pkg.set('path1/asdf', LOCAL_MANIFEST)
pkg.set('path1/qwer', LOCAL_MANIFEST)
pkg.set('path2/first/asdf', LOCAL_MANIFEST)
pkg.set('path2/second/asdf', LOCAL_MANIFEST)
assert repr(pkg) == TEST_REPR
def test_remote_repr(self):
with patch('t4.packages.get_size_and_meta',
return_value=(0, dict(), '0')):
TEST_REPR = ("(remote Package)\n" " └─asdf\n")
pkg = Package()
pkg.set('asdf', 's3://my-bucket/asdf')
assert repr(pkg) == TEST_REPR
TEST_REPR = ("(remote Package)\n" " └─asdf\n" " └─qwer\n")
pkg = Package()
pkg.set('asdf', 's3://my-bucket/asdf')
pkg.set('qwer', LOCAL_MANIFEST)
assert repr(pkg) == TEST_REPR
def test_filter(self):
pkg = Package()
pkg.set('a/df', LOCAL_MANIFEST)
pkg.set('a/qw', LOCAL_MANIFEST)
p_copy = pkg.filter(lambda lk, entry: lk == 'a/df')
assert list(p_copy) == ['a'] and list(p_copy['a']) == ['df']
pkg = Package()
pkg.set('a/df', LOCAL_MANIFEST)
pkg.set('a/qw', LOCAL_MANIFEST)
pkg.set('b/df', LOCAL_MANIFEST)
pkg['a'].set_meta({'foo': 'bar'})
pkg['b'].set_meta({'foo': 'bar'})
p_copy = pkg.filter(lambda lk, entry: lk == 'a/',
include_directories=True)
assert list(p_copy) == []
p_copy = pkg.filter(lambda lk, entry: lk == 'a/' or lk == 'a/df',
include_directories=True)
assert list(p_copy) == ['a'] and list(p_copy['a']) == ['df']
def test_siblings_succeed():
pkg = Package()
pkg.set('as/df', LOCAL_MANIFEST)
pkg.set('as/qw', LOCAL_MANIFEST)
def test_overwrite_entry_fails():
with pytest.raises(QuiltException):
pkg = Package()
pkg.set('asdf', LOCAL_MANIFEST)
pkg.set('asdf/jkl', LOCAL_MANIFEST)
def test_invalid_key(self):
pkg = Package()
with pytest.raises(QuiltException):
pkg.set('', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('foo/', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('foo', './')
with pytest.raises(QuiltException):
pkg.set('foo', os.path.dirname(__file__))
# we do not allow '.' or '..' files or filename separators
with pytest.raises(QuiltException):
pkg.set('.', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('..', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('./foo', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('../foo', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('foo/.', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('foo/..', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('foo/./bar', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('foo/../bar', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('s3://foo/.', LOCAL_MANIFEST)
with pytest.raises(QuiltException):
pkg.set('s3://foo/..', LOCAL_MANIFEST)
def test_invalid_set_key(tmpdir):
"""Verify an exception when setting a key with a path object."""
pkg = Package()
with pytest.raises(TypeError):
pkg.set('asdf/jkl', 123)