def test_brackets(): pkg = Package() pkg.set('asdf/jkl', LOCAL_MANIFEST) pkg.set('asdf/qwer', LOCAL_MANIFEST) pkg.set('qwer/asdf', LOCAL_MANIFEST) assert set(pkg.keys()) == {'asdf', 'qwer'} pkg2 = pkg['asdf'] assert set(pkg2.keys()) == {'jkl', 'qwer'} assert pkg['asdf']['qwer'].get() == pathlib.Path(LOCAL_MANIFEST).as_uri() assert pkg['asdf']['qwer'] == pkg['asdf/qwer'] == pkg[('asdf', 'qwer')] assert pkg[[]] == pkg pkg = (Package().set( 'foo', os.path.join(os.path.dirname(__file__), 'data', 'foo.txt'), {'foo': 'blah'})) pkg['foo'].meta['target'] = 'unicode' pkg.build() assert pkg['foo'].deserialize() == '123\n' assert pkg['foo']() == '123\n' with pytest.raises(KeyError): pkg['baz'] with pytest.raises(TypeError): pkg[b'asdf'] with pytest.raises(TypeError): pkg[0]
def test_build(tmpdir): """Verify that build dumps the manifest to appdirs directory.""" new_pkg = Package() # Create a dummy file to add to the package. test_file_name = 'bar' with open(test_file_name, "w") as fd: fd.write('test_file_content_string') test_file = Path(fd.name) # Build a new package into the local registry. new_pkg = new_pkg.set('foo', test_file_name) top_hash = new_pkg.build("Quilt/Test") # Verify manifest is registered by hash. out_path = Path(BASE_PATH, ".quilt/packages", top_hash) with open(out_path) as fd: pkg = Package.load(fd) assert test_file.resolve().as_uri() == pkg['foo'].physical_keys[0] # Verify latest points to the new location. named_pointer_path = Path(BASE_PATH, ".quilt/named_packages/Quilt/Test/latest") with open(named_pointer_path) as fd: assert fd.read().replace('\n', '') == top_hash # Test unnamed packages. new_pkg = Package() new_pkg = new_pkg.set('bar', test_file_name) top_hash = new_pkg.build() out_path = Path(BASE_PATH, ".quilt/packages", top_hash) with open(out_path) as fd: pkg = Package.load(fd) assert test_file.resolve().as_uri() == pkg['bar'].physical_keys[0]
def test_brackets(self): pkg = Package() pkg.set('asdf/jkl', LOCAL_MANIFEST) pkg.set('asdf/qwer', LOCAL_MANIFEST) pkg.set('qwer/asdf', LOCAL_MANIFEST) assert set(pkg.keys()) == {'asdf', 'qwer'} pkg2 = pkg['asdf'] assert set(pkg2.keys()) == {'jkl', 'qwer'} assert pkg['asdf']['qwer'].get() == LOCAL_MANIFEST.as_uri() assert pkg['asdf']['qwer'] == pkg['asdf/qwer'] == pkg[('asdf', 'qwer')] assert pkg[[]] == pkg pkg = (Package().set('foo', DATA_DIR / 'foo.txt', {'foo': 'blah'})) pkg['foo'].meta['target'] = 'unicode' pkg.build() assert pkg['foo'].deserialize() == '123\n' assert pkg['foo']() == '123\n' with pytest.raises(KeyError): pkg['baz'] with pytest.raises(TypeError): pkg[b'asdf'] with pytest.raises(TypeError): pkg[0]
def test_manifest(self): pkg = Package() pkg.set('as/df', LOCAL_MANIFEST) pkg.set('as/qw', LOCAL_MANIFEST) top_hash = pkg.build().top_hash manifest = list(pkg.manifest) pkg2 = Package.browse(top_hash=top_hash, registry='local') assert list(pkg.manifest) == list(pkg2.manifest)
def test_manifest(): pkg = Package() pkg.set('as/df', LOCAL_MANIFEST) pkg.set('as/qw', LOCAL_MANIFEST) top_hash = pkg.build() manifest = list(pkg.manifest) pkg2 = Package.browse(pkg_hash=top_hash) assert list(pkg.manifest) == list(pkg2.manifest)
def test_map(): pkg = Package() pkg.set('as/df', LOCAL_MANIFEST) pkg.set('as/qw', LOCAL_MANIFEST) assert set(pkg.map(lambda lk, entry: lk)) == {'as/df', 'as/qw'} pkg['as'].set_meta({'foo': 'bar'}) assert set(pkg.map(lambda lk, entry: lk, include_directories=True)) ==\ {'as/df', 'as/qw', 'as/'}
def test_iter(): pkg = Package() assert not pkg pkg.set('asdf', LOCAL_MANIFEST) assert list(pkg) == ['asdf'] pkg.set('jkl;', REMOTE_MANIFEST) assert set(pkg) == {'asdf', 'jkl;'}
def test_keys(): pkg = Package() assert not pkg.keys() pkg.set('asdf', LOCAL_MANIFEST) assert set(pkg.keys()) == {'asdf'} pkg.set('jkl;', REMOTE_MANIFEST) assert set(pkg.keys()) == {'asdf', 'jkl;'} pkg.delete('asdf') assert set(pkg.keys()) == {'jkl;'}
def test_default_registry(self): new_pkg = Package() # Create a dummy file to add to the package. test_file_name = 'bar' with open(test_file_name, "w") as fd: fd.write('test_file_content_string') test_file = Path(fd.name) # Build a new package into the local registry. new_pkg = new_pkg.set('foo', test_file_name) top_hash = new_pkg.build("Quilt/Test").top_hash # Verify manifest is registered by hash. out_path = Path(BASE_PATH, ".quilt/packages", top_hash) with open(out_path) as fd: pkg = Package.load(fd) assert test_file.resolve().as_uri() == pkg['foo'].physical_keys[0] # Verify latest points to the new location. named_pointer_path = Path(BASE_PATH, ".quilt/named_packages/Quilt/Test/latest") with open(named_pointer_path) as fd: assert fd.read().replace('\n', '') == top_hash # Test unnamed packages. new_pkg = Package() new_pkg = new_pkg.set('bar', test_file_name) top_hash = new_pkg.build().top_hash out_path = Path(BASE_PATH, ".quilt/packages", top_hash) with open(out_path) as fd: pkg = Package.load(fd) assert test_file.resolve().as_uri() == pkg['bar'].physical_keys[0] new_base_path = Path(BASE_PATH, ".quilttest") with patch('t4.packages.get_from_config') as mock_config: mock_config.return_value = new_base_path top_hash = new_pkg.build("Quilt/Test").top_hash out_path = Path(new_base_path, ".quilt/packages", top_hash).resolve() with open(out_path) as fd: pkg = Package.load(fd) assert test_file.resolve().as_uri( ) == pkg['bar'].physical_keys[0] with patch('t4.packages.get_from_config') as mock_config: mock_config.return_value = new_base_path new_pkg.push("Quilt/Test") with open(out_path) as fd: pkg = Package.load(fd) assert pkg['bar'].physical_keys[0].endswith( '.quilttest/Quilt/Test/bar')
def test_long_repr(): pkg = Package() for i in range(30): pkg.set('path{}/asdf'.format(i), LOCAL_MANIFEST) r = repr(pkg) assert r.count('\n') == 20 assert r[-4:] == '...\n' pkg = Package() for i in range(10): pkg.set('path{}/asdf'.format(i), LOCAL_MANIFEST) pkg.set('path{}/qwer'.format(i), LOCAL_MANIFEST) pkgrepr = repr(pkg) assert pkgrepr.count('\n') == 20 assert pkgrepr.find('path9/') > 0
def test_filter(): pkg = Package() pkg.set('as/df', LOCAL_MANIFEST) pkg.set('as/qw', LOCAL_MANIFEST) assert list(pkg.filter(lambda lk, entry: lk == 'as/df')) == [ ('as/df', pkg['as/df']) ] pkg['as'].set_meta({'foo': 'bar'}) assert list(pkg.filter(lambda lk, entry: lk == 'as/df')) == [ ('as/df', pkg['as/df']) ] assert list( pkg.filter(lambda lk, entry: lk == 'as/', include_directories=True)) == [('as/', pkg['as'])]
def test_local_push(tmpdir): """ Verify loading local manifest and data into S3. """ with patch('t4.packages.put_bytes') as bytes_mock, \ patch('t4.data_transfer._copy_local_file') as file_mock, \ patch('t4.packages.get_remote_registry') as config_mock: config_mock.return_value = tmpdir / 'package_contents' new_pkg = Package() contents = 'blah' test_file = pathlib.Path(tmpdir) / 'bar' test_file.write_text(contents) new_pkg = new_pkg.set('foo', test_file) new_pkg.push('Quilt/package', tmpdir / 'package_contents') push_uri = pathlib.Path(tmpdir, 'package_contents').as_uri() # Manifest copied top_hash = new_pkg.top_hash() bytes_mock.assert_any_call( top_hash.encode(), push_uri + '/.quilt/named_packages/Quilt/package/latest') bytes_mock.assert_any_call(ANY, push_uri + '/.quilt/packages/' + top_hash) # Data copied file_mock.assert_called_once_with( ANY, len(contents), str(test_file), str(tmpdir / 'package_contents/Quilt/package/foo'), {})
def test_load_into_t4(tmpdir): """ Verify loading local manifest and data into S3. """ with patch('t4.packages.put_bytes') as bytes_mock, \ patch('t4.data_transfer._upload_file') as file_mock, \ patch('t4.packages.get_remote_registry') as config_mock: config_mock.return_value = 's3://my_test_bucket' new_pkg = Package() # Create a dummy file to add to the package. contents = 'blah' test_file = pathlib.Path(tmpdir) / 'bar' test_file.write_text(contents) new_pkg = new_pkg.set('foo', test_file) new_pkg.push('Quilt/package', 's3://my_test_bucket/') # Manifest copied top_hash = new_pkg.top_hash() bytes_mock.assert_any_call( top_hash.encode(), 's3://my_test_bucket/.quilt/named_packages/Quilt/package/latest') bytes_mock.assert_any_call( ANY, 's3://my_test_bucket/.quilt/packages/' + top_hash) # Data copied file_mock.assert_called_once_with(ANY, len(contents), str(test_file), 'my_test_bucket', 'Quilt/package/foo', {})
def test_load_into_t4(self): """ Verify loading local manifest and data into S3. """ top_hash = '5333a204bbc6e21607c2bc842f4a77d2e21aa6147cf2bf493dbf6282188d01ca' self.s3_stubber.add_response(method='put_object', service_response={'VersionId': 'v1'}, expected_params={ 'Body': ANY, 'Bucket': 'my_test_bucket', 'Key': 'Quilt/package/foo', 'Metadata': { 'helium': '{}' } }) self.s3_stubber.add_response(method='put_object', service_response={'VersionId': 'v2'}, expected_params={ 'Body': ANY, 'Bucket': 'my_test_bucket', 'Key': '.quilt/packages/' + top_hash, 'Metadata': { 'helium': 'null' } }) self.s3_stubber.add_response( method='put_object', service_response={'VersionId': 'v3'}, expected_params={ 'Body': top_hash.encode(), 'Bucket': 'my_test_bucket', 'Key': '.quilt/named_packages/Quilt/package/1234567890', 'Metadata': { 'helium': 'null' } }) self.s3_stubber.add_response( method='put_object', service_response={'VersionId': 'v4'}, expected_params={ 'Body': top_hash.encode(), 'Bucket': 'my_test_bucket', 'Key': '.quilt/named_packages/Quilt/package/latest', 'Metadata': { 'helium': 'null' } }) new_pkg = Package() # Create a dummy file to add to the package. contents = 'blah' test_file = Path('bar') test_file.write_text(contents) new_pkg = new_pkg.set('foo', test_file) with patch('time.time', return_value=1234567890): new_pkg.push('Quilt/package', 's3://my_test_bucket/')
def test_reduce(): pkg = Package() pkg.set('as/df', LOCAL_MANIFEST) pkg.set('as/qw', LOCAL_MANIFEST) assert pkg.reduce(lambda a, b: a) == ('as/df', pkg['as/df']) assert pkg.reduce(lambda a, b: b) == ('as/qw', pkg['as/qw']) assert list(pkg.reduce(lambda a, b: a + [b], [])) == [('as/df', pkg['as/df']), ('as/qw', pkg['as/qw'])] pkg['as'].set_meta({'foo': 'bar'}) assert pkg.reduce(lambda a, b: b, include_directories=True) ==\ ('as/qw', pkg['as/qw']) assert list(pkg.reduce(lambda a, b: a + [b], [], include_directories=True)) == [ ('as/', pkg['as']), ('as/df', pkg['as/df']), ('as/qw', pkg['as/qw']) ]
def test_tophash_changes(tmpdir): test_file = tmpdir / 'test.txt' test_file.write_text('asdf', 'utf-8') pkg = Package() th1 = pkg.top_hash() pkg.set('asdf', test_file) pkg.build() th2 = pkg.top_hash() assert th1 != th2 test_file.write_text('jkl', 'utf-8') pkg.set('jkl', test_file) pkg.build() th3 = pkg.top_hash() assert th1 != th3 assert th2 != th3 pkg.delete('jkl') th4 = pkg.top_hash() assert th2 == th4
def test_dir_meta(tmpdir): test_meta = {'test': 'meta'} pkg = Package() pkg.set('asdf/jkl', LOCAL_MANIFEST) pkg.set('asdf/qwer', LOCAL_MANIFEST) pkg.set('qwer/asdf', LOCAL_MANIFEST) pkg.set('qwer/as/df', LOCAL_MANIFEST) pkg.build() assert pkg['asdf'].get_meta() == {} assert pkg.get_meta() == {} assert pkg['qwer']['as'].get_meta() == {} pkg['asdf'].set_meta(test_meta) assert pkg['asdf'].get_meta() == test_meta pkg['qwer']['as'].set_meta(test_meta) assert pkg['qwer']['as'].get_meta() == test_meta pkg.set_meta(test_meta) assert pkg.get_meta() == test_meta dump_path = os.path.join(tmpdir, 'test_meta') with open(dump_path, 'w') as f: pkg.dump(f) with open(dump_path) as f: pkg2 = Package.load(f) assert pkg2['asdf'].get_meta() == test_meta assert pkg2['qwer']['as'].get_meta() == test_meta assert pkg2.get_meta() == test_meta
def test_diff(): new_pkg = Package() # Create a dummy file to add to the package. test_file_name = 'bar' with open(test_file_name, "w") as fd: fd.write('test_file_content_string') test_file = Path(fd.name) # Build a new package into the local registry. new_pkg = new_pkg.set('foo', test_file_name) top_hash = new_pkg.build("Quilt/Test") p1 = Package.browse('Quilt/Test') p2 = Package.browse('Quilt/Test') assert p1.diff(p2) == ([], [], [])
def test_local_push(self): """ Verify loading local manifest and data into a local dir. """ top_hash = '5333a204bbc6e21607c2bc842f4a77d2e21aa6147cf2bf493dbf6282188d01ca' new_pkg = Package() contents = 'blah' test_file = Path('bar') test_file.write_text(contents) new_pkg = new_pkg.set('foo', test_file) new_pkg.push('Quilt/package', 'package_contents') push_dir = Path('package_contents') assert (push_dir / '.quilt/named_packages/Quilt/package/latest' ).read_text() == top_hash assert (push_dir / ('.quilt/packages/' + top_hash)).exists() assert (push_dir / 'Quilt/package/foo').read_text() == contents
def test_repr(): TEST_REPR = ("asdf\n" "path1/\n" " asdf\n" " qwer\n" "path2/\n" " first/\n" " asdf\n" " second/\n" " asdf\n" "qwer\n") pkg = Package() pkg.set('asdf', LOCAL_MANIFEST) pkg.set('qwer', LOCAL_MANIFEST) pkg.set('path1/asdf', LOCAL_MANIFEST) pkg.set('path1/qwer', LOCAL_MANIFEST) pkg.set('path2/first/asdf', LOCAL_MANIFEST) pkg.set('path2/second/asdf', LOCAL_MANIFEST) assert repr(pkg) == TEST_REPR
def test_local_repr(self): TEST_REPR = ("(local Package)\n" " └─asdf\n" " └─path1/\n" " └─asdf\n" " └─qwer\n" " └─path2/\n" " └─first/\n" " └─asdf\n" " └─second/\n" " └─asdf\n" " └─qwer\n") pkg = Package() pkg.set('asdf', LOCAL_MANIFEST) pkg.set('qwer', LOCAL_MANIFEST) pkg.set('path1/asdf', LOCAL_MANIFEST) pkg.set('path1/qwer', LOCAL_MANIFEST) pkg.set('path2/first/asdf', LOCAL_MANIFEST) pkg.set('path2/second/asdf', LOCAL_MANIFEST) assert repr(pkg) == TEST_REPR
def test_remote_repr(self): with patch('t4.packages.get_size_and_meta', return_value=(0, dict(), '0')): TEST_REPR = ("(remote Package)\n" " └─asdf\n") pkg = Package() pkg.set('asdf', 's3://my-bucket/asdf') assert repr(pkg) == TEST_REPR TEST_REPR = ("(remote Package)\n" " └─asdf\n" " └─qwer\n") pkg = Package() pkg.set('asdf', 's3://my-bucket/asdf') pkg.set('qwer', LOCAL_MANIFEST) assert repr(pkg) == TEST_REPR
def test_filter(self): pkg = Package() pkg.set('a/df', LOCAL_MANIFEST) pkg.set('a/qw', LOCAL_MANIFEST) p_copy = pkg.filter(lambda lk, entry: lk == 'a/df') assert list(p_copy) == ['a'] and list(p_copy['a']) == ['df'] pkg = Package() pkg.set('a/df', LOCAL_MANIFEST) pkg.set('a/qw', LOCAL_MANIFEST) pkg.set('b/df', LOCAL_MANIFEST) pkg['a'].set_meta({'foo': 'bar'}) pkg['b'].set_meta({'foo': 'bar'}) p_copy = pkg.filter(lambda lk, entry: lk == 'a/', include_directories=True) assert list(p_copy) == [] p_copy = pkg.filter(lambda lk, entry: lk == 'a/' or lk == 'a/df', include_directories=True) assert list(p_copy) == ['a'] and list(p_copy['a']) == ['df']
def test_siblings_succeed(): pkg = Package() pkg.set('as/df', LOCAL_MANIFEST) pkg.set('as/qw', LOCAL_MANIFEST)
def test_overwrite_entry_fails(): with pytest.raises(QuiltException): pkg = Package() pkg.set('asdf', LOCAL_MANIFEST) pkg.set('asdf/jkl', LOCAL_MANIFEST)
def test_invalid_key(self): pkg = Package() with pytest.raises(QuiltException): pkg.set('', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('foo/', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('foo', './') with pytest.raises(QuiltException): pkg.set('foo', os.path.dirname(__file__)) # we do not allow '.' or '..' files or filename separators with pytest.raises(QuiltException): pkg.set('.', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('..', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('./foo', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('../foo', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('foo/.', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('foo/..', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('foo/./bar', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('foo/../bar', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('s3://foo/.', LOCAL_MANIFEST) with pytest.raises(QuiltException): pkg.set('s3://foo/..', LOCAL_MANIFEST)
def test_invalid_set_key(tmpdir): """Verify an exception when setting a key with a path object.""" pkg = Package() with pytest.raises(TypeError): pkg.set('asdf/jkl', 123)