def enqueue(self, filename, compress=False, **metadata_fields): '''enqueue a file with the specified metadata to be pushed Args: filename: the file to enqueue compress: whether or not to compress the file before enqueueing Returns the File with complete metadata that will be pushed. ''' log.info('Enqueing ' + filename) if compress: try: f = File.from_filename_compressed(filename, **metadata_fields) except OverflowError: log.warning('Compression failed. Falling back to uncompressed ' 'uploads') f = File.from_filename(filename, **metadata_fields) else: f = File.from_filename(filename, **metadata_fields) fname = f.metadata['id'] + '.tar' dest = os.path.join(self.queue_dir, fname) f.to_bundle(dest) return f
def enqueue(self, filename, compress=False, **metadata_fields): '''enqueue a file with the specified metadata to be pushed Args: filename: the file to enqueue compress: whether or not to compress the file before enqueueing Returns the File with complete metadata that will be pushed. ''' log.info('Enqueing ' + filename) if compress: try: f = File.from_filename_compressed(filename, **metadata_fields) except OverflowError: log.warning('Compression failed. Falling back to uncompressed ' 'uploads') f = File.from_filename(filename, **metadata_fields) else: f = File.from_filename(filename, **metadata_fields) fname = f.metadata['id'] + '.tar' dest = os.path.join(self.queue_dir, fname) f.to_bundle(dest) return f
def random_file(tmpdir, metadata=None): name = random_word(10) content = random_word(256) f = tmpdir.join(name) f.write(content) if metadata is None: metadata = generate_random_metadata() return File.from_filename(f.strpath, **metadata)
def random_file(tmpdir, metadata=None): name = random_word(10) content = random_word(256) f = tmpdir.join(name) f.write(content) if metadata is None: metadata = random_metadata() return File.from_filename(f.strpath, **metadata)
def prepare_metadata_and_push(self, filename, **metadata_fields): '''push a file to the archive with the specified metadata Args: filename: path of the file to push metadata_fields: metadata fields for file. Missing fields will be added if they can be determined. Othwerise, InvalidDatalakeMetadata will be raised. returns the url to which the file was pushed. ''' f = File.from_filename(filename, **metadata_fields) return self.push(f)
def enqueue(self, filename, compress=False, **metadata_fields): '''enqueue a file with the specified metadata to be pushed Args: filename: the file to enqueue compress: whether or not to compress the file before enqueueing Returns the File with complete metadata that will be pushed. ''' log.info('Enqueing ' + filename) if compress: f = File.from_filename_compressed(filename, **metadata_fields) else: f = File.from_filename(filename, **metadata_fields) fname = f.metadata['id'] + '.tar' dest = os.path.join(self.queue_dir, fname) f.to_bundle(dest) return f
def test_non_existent_file(): with pytest.raises(IOError): File.from_filename('surelythisfiledoesnotexist.txt')
def test_non_existent_file(): with pytest.raises(IOError): File.from_filename('surelythisfiledoesnotexist.txt')