def enqueue(self, filename, compress=False, **metadata_fields): '''enqueue a file with the specified metadata to be pushed Args: filename: the file to enqueue compress: whether or not to compress the file before enqueueing Returns the File with complete metadata that will be pushed. ''' log.info('Enqueing ' + filename) if compress: try: f = File.from_filename_compressed(filename, **metadata_fields) except OverflowError: log.warning('Compression failed. Falling back to uncompressed ' 'uploads') f = File.from_filename(filename, **metadata_fields) else: f = File.from_filename(filename, **metadata_fields) fname = f.metadata['id'] + '.tar' dest = os.path.join(self.queue_dir, fname) f.to_bundle(dest) return f
def enqueue(self, filename, compress=False, **metadata_fields): '''enqueue a file with the specified metadata to be pushed Args: filename: the file to enqueue compress: whether or not to compress the file before enqueueing Returns the File with complete metadata that will be pushed. ''' log.info('Enqueing ' + filename) if compress: try: f = File.from_filename_compressed(filename, **metadata_fields) except OverflowError: log.warning('Compression failed. Falling back to uncompressed ' 'uploads') f = File.from_filename(filename, **metadata_fields) else: f = File.from_filename(filename, **metadata_fields) fname = f.metadata['id'] + '.tar' dest = os.path.join(self.queue_dir, fname) f.to_bundle(dest) return f
def test_bundle_with_invalid_metadata(bundle_maker, random_metadata): del (random_metadata['what']) m = json.dumps(random_metadata).encode('utf-8') b = bundle_maker(content='1234'.encode('utf-8'), metadata=m, version='0'.encode('utf-8')) with pytest.raises(InvalidDatalakeMetadata): File.from_bundle(b)
def test_bundle_with_invalid_metadata(bundle_maker, random_metadata): del(random_metadata['what']) m = json.dumps(random_metadata).encode('utf-8') b = bundle_maker(content='1234'.encode('utf-8'), metadata=m, version='0'.encode('utf-8')) with pytest.raises(InvalidDatalakeMetadata): File.from_bundle(b)
def random_file(tmpdir, metadata=None): name = random_word(10) content = random_word(256) f = tmpdir.join(name) f.write(content) if metadata is None: metadata = random_metadata() return File.from_filename(f.strpath, **metadata)
def random_file(tmpdir, metadata=None): name = random_word(10) content = random_word(256) f = tmpdir.join(name) f.write(content) if metadata is None: metadata = generate_random_metadata() return File.from_filename(f.strpath, **metadata)
def _fetch_s3_url(self, url, stream=False): k = self._get_key_from_url(url) m = self._get_metadata_from_key(k) if stream: return StreamingFile(k, **m) fd = BytesIO() k.get_contents_to_file(fd) fd.seek(0) return File(fd, **m)
def _fetch_http_url(self, url, stream=False): m = self._get_metadata_from_http_url(url) k = self._stream_http_url(url) if stream: return StreamingHTTPFile(k, **m) fd = BytesIO() for block in k.iter_content(1024): fd.write(block) fd.seek(0) return File(fd, **m)
def test_valid_bundle(tmpdir, random_metadata): p = os.path.join(str(tmpdir), 'foo.tar') f1 = random_file(tmpdir, metadata=random_metadata) f1.to_bundle(p) f2 = File.from_bundle(p) assert f1.metadata == f2.metadata content1 = f1.read() content2 = f2.read() assert content1 assert content1 == content2
def test_valid_bundle(tmpdir, random_metadata): p = os.path.join(str(tmpdir), 'foo.tar') f1 = random_file(tmpdir, metadata=random_metadata) f1.to_bundle(p) f2 = File.from_bundle(p) assert f1.metadata == f2.metadata content1 = f1.read() content2 = f2.read() assert content1 assert content1 == content2
def enqueue(self, filename, compress=False, **metadata_fields): '''enqueue a file with the specified metadata to be pushed Args: filename: the file to enqueue compress: whether or not to compress the file before enqueueing Returns the File with complete metadata that will be pushed. ''' log.info('Enqueing ' + filename) if compress: f = File.from_filename_compressed(filename, **metadata_fields) else: f = File.from_filename(filename, **metadata_fields) fname = f.metadata['id'] + '.tar' dest = os.path.join(self.queue_dir, fname) f.to_bundle(dest) return f
def _synchronous_push(self, filename): try: f = File.from_bundle(filename) except InvalidDatalakeBundle as e: msg = '{}. Skipping upload.'.format(e.args[0]) log.exception(msg) return url = self._archive.push(f) msg = 'Pushed {}({}) to {}'.format(filename, f.metadata['path'], url) log.info(msg) os.unlink(filename) if self._callback is not None: self._callback(filename)
def _synchronous_push(self, filename): try: f = File.from_bundle(filename) except InvalidDatalakeBundle as e: msg = '{}. Skipping upload.'.format(e.args[0]) log.exception(msg) return url = self._archive.push(f) msg = 'Pushed {}({}) to {}'.format(filename, f.metadata['path'], url) log.info(msg) os.unlink(filename) if self._callback is not None: self._callback(filename)
def prepare_metadata_and_push(self, filename, **metadata_fields): '''push a file to the archive with the specified metadata Args: filename: path of the file to push metadata_fields: metadata fields for file. Missing fields will be added if they can be determined. Othwerise, InvalidDatalakeMetadata will be raised. returns the url to which the file was pushed. ''' f = File.from_filename(filename, **metadata_fields) return self.push(f)
def test_pre_python_3_bundle(): # prior to python 3 support, we relied on python to choose the most # suitable encoding for files. Now we do it explicitly. Make sure legacy # bundles work. eyedee = '7c72f3ab092445a08aa6983c864c087c' expected_content = b'Wake up.\nEat. Mmm.\nHappy hour.\nSleep.\n' expected_metadata = { 'end': 1474308636507, 'hash': '70373dec2de49d566fc1e34bacca7561', 'id': eyedee, 'path': '/home/brian/src/datalake/chicken.log', 'start': 1474308548000, 'version': 0, 'what': 'chicken', 'where': 'nomad', 'work_id': None } b = os.path.join(legacy_bundles, eyedee + '.tar') f = File.from_bundle(b) assert f.metadata == expected_metadata assert f.read() == expected_content
def test_pre_python_3_bundle(): # prior to python 3 support, we relied on python to choose the most # suitable encoding for files. Now we do it explicitly. Make sure legacy # bundles work. eyedee = '7c72f3ab092445a08aa6983c864c087c' expected_content = b'Wake up.\nEat. Mmm.\nHappy hour.\nSleep.\n' expected_metadata = { 'end': 1474308636507, 'hash': '70373dec2de49d566fc1e34bacca7561', 'id': eyedee, 'path': '/home/brian/src/datalake/chicken.log', 'start': 1474308548000, 'version': 0, 'what': 'chicken', 'where': 'nomad', 'work_id': None } b = os.path.join(legacy_bundles, eyedee + '.tar') f = File.from_bundle(b) assert f.metadata == expected_metadata assert f.read() == expected_content
def test_bundle_not_tar(tmpfile): f = tmpfile('foobar') with pytest.raises(InvalidDatalakeBundle): File.from_bundle(f)
def test_bundle_without_content(bundle_maker, random_metadata): m = json.dumps(random_metadata).encode('utf-8') b = bundle_maker(metadata=m, version='0'.encode('utf-8')) with pytest.raises(InvalidDatalakeBundle): File.from_bundle(b)
def test_non_existent_file(): with pytest.raises(IOError): File.from_filename('surelythisfiledoesnotexist.txt')
def test_bundle_with_non_json_metadata(bundle_maker): b = bundle_maker(content='1234'.encode('utf-8'), metadata='not:a%json#'.encode('utf-8'), version='0'.encode('utf-8')) with pytest.raises(InvalidDatalakeBundle): File.from_bundle(b)
def test_bundle_with_non_json_metadata(bundle_maker): b = bundle_maker(content='1234'.encode('utf-8'), metadata='not:a%json#'.encode('utf-8'), version='0'.encode('utf-8')) with pytest.raises(InvalidDatalakeBundle): File.from_bundle(b)
def test_bundle_without_content(bundle_maker, random_metadata): m = json.dumps(random_metadata).encode('utf-8') b = bundle_maker(metadata=m, version='0'.encode('utf-8')) with pytest.raises(InvalidDatalakeBundle): File.from_bundle(b)
def test_bundle_without_version(bundle_maker, random_metadata): m = json.dumps(random_metadata).encode('utf-8') b = bundle_maker(content='1234'.encode('utf-8'), metadata=m) with pytest.raises(InvalidDatalakeBundle): File.from_bundle(b)
def test_non_existent_file(): with pytest.raises(IOError): File.from_filename('surelythisfiledoesnotexist.txt')
def test_bundle_not_tar(tmpfile): f = tmpfile('foobar') with pytest.raises(InvalidDatalakeBundle): File.from_bundle(f)
def test_bundle_without_version(bundle_maker, random_metadata): m = json.dumps(random_metadata).encode('utf-8') b = bundle_maker(content='1234'.encode('utf-8'), metadata=m) with pytest.raises(InvalidDatalakeBundle): File.from_bundle(b)