def test_bundle_with_invalid_metadata(bundle_maker, random_metadata):
del(random_metadata['what'])
m = json.dumps(random_metadata).encode('utf-8')
b = bundle_maker(content='1234'.encode('utf-8'),
metadata=m,
version='0'.encode('utf-8'))
with pytest.raises(InvalidDatalakeMetadata):
File.from_bundle(b)
def test_bundle_with_invalid_metadata(bundle_maker, random_metadata):
del (random_metadata['what'])
m = json.dumps(random_metadata).encode('utf-8')
b = bundle_maker(content='1234'.encode('utf-8'),
metadata=m,
version='0'.encode('utf-8'))
with pytest.raises(InvalidDatalakeMetadata):
File.from_bundle(b)
def test_valid_bundle(tmpdir, random_metadata):
p = os.path.join(str(tmpdir), 'foo.tar')
f1 = random_file(tmpdir, metadata=random_metadata)
f1.to_bundle(p)
f2 = File.from_bundle(p)
assert f1.metadata == f2.metadata
content1 = f1.read()
content2 = f2.read()
assert content1
assert content1 == content2
def test_valid_bundle(tmpdir, random_metadata):
p = os.path.join(str(tmpdir), 'foo.tar')
f1 = random_file(tmpdir, metadata=random_metadata)
f1.to_bundle(p)
f2 = File.from_bundle(p)
assert f1.metadata == f2.metadata
content1 = f1.read()
content2 = f2.read()
assert content1
assert content1 == content2
def _synchronous_push(self, filename):
try:
f = File.from_bundle(filename)
except InvalidDatalakeBundle as e:
msg = '{}. Skipping upload.'.format(e.args[0])
log.exception(msg)
return
url = self._archive.push(f)
msg = 'Pushed {}({}) to {}'.format(filename, f.metadata['path'], url)
log.info(msg)
os.unlink(filename)
if self._callback is not None:
self._callback(filename)
def _synchronous_push(self, filename):
try:
f = File.from_bundle(filename)
except InvalidDatalakeBundle as e:
msg = '{}. Skipping upload.'.format(e.args[0])
log.exception(msg)
return
url = self._archive.push(f)
msg = 'Pushed {}({}) to {}'.format(filename, f.metadata['path'], url)
log.info(msg)
os.unlink(filename)
if self._callback is not None:
self._callback(filename)
def test_pre_python_3_bundle():
# prior to python 3 support, we relied on python to choose the most
# suitable encoding for files. Now we do it explicitly. Make sure legacy
# bundles work.
eyedee = '7c72f3ab092445a08aa6983c864c087c'
expected_content = b'Wake up.\nEat. Mmm.\nHappy hour.\nSleep.\n'
expected_metadata = {
'end': 1474308636507,
'hash': '70373dec2de49d566fc1e34bacca7561',
'id': eyedee,
'path': '/home/brian/src/datalake/chicken.log',
'start': 1474308548000,
'version': 0,
'what': 'chicken',
'where': 'nomad',
'work_id': None
}
b = os.path.join(legacy_bundles, eyedee + '.tar')
f = File.from_bundle(b)
assert f.metadata == expected_metadata
assert f.read() == expected_content
def test_pre_python_3_bundle():
# prior to python 3 support, we relied on python to choose the most
# suitable encoding for files. Now we do it explicitly. Make sure legacy
# bundles work.
eyedee = '7c72f3ab092445a08aa6983c864c087c'
expected_content = b'Wake up.\nEat. Mmm.\nHappy hour.\nSleep.\n'
expected_metadata = {
'end': 1474308636507,
'hash': '70373dec2de49d566fc1e34bacca7561',
'id': eyedee,
'path': '/home/brian/src/datalake/chicken.log',
'start': 1474308548000,
'version': 0,
'what': 'chicken',
'where': 'nomad',
'work_id': None
}
b = os.path.join(legacy_bundles, eyedee + '.tar')
f = File.from_bundle(b)
assert f.metadata == expected_metadata
assert f.read() == expected_content
def test_bundle_not_tar(tmpfile):
f = tmpfile('foobar')
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(f)
def test_bundle_with_non_json_metadata(bundle_maker):
b = bundle_maker(content='1234'.encode('utf-8'),
metadata='not:a%json#'.encode('utf-8'),
version='0'.encode('utf-8'))
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(b)
def test_bundle_without_content(bundle_maker, random_metadata):
m = json.dumps(random_metadata).encode('utf-8')
b = bundle_maker(metadata=m, version='0'.encode('utf-8'))
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(b)
def test_bundle_without_version(bundle_maker, random_metadata):
m = json.dumps(random_metadata).encode('utf-8')
b = bundle_maker(content='1234'.encode('utf-8'), metadata=m)
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(b)
def test_bundle_not_tar(tmpfile):
f = tmpfile('foobar')
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(f)
def test_bundle_with_non_json_metadata(bundle_maker):
b = bundle_maker(content='1234'.encode('utf-8'),
metadata='not:a%json#'.encode('utf-8'),
version='0'.encode('utf-8'))
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(b)
def test_bundle_without_content(bundle_maker, random_metadata):
m = json.dumps(random_metadata).encode('utf-8')
b = bundle_maker(metadata=m, version='0'.encode('utf-8'))
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(b)
def test_bundle_without_version(bundle_maker, random_metadata):
m = json.dumps(random_metadata).encode('utf-8')
b = bundle_maker(content='1234'.encode('utf-8'), metadata=m)
with pytest.raises(InvalidDatalakeBundle):
File.from_bundle(b)