def enqueue(self, filename, compress=False, **metadata_fields):
'''enqueue a file with the specified metadata to be pushed
Args:
filename: the file to enqueue
compress: whether or not to compress the file before enqueueing
Returns the File with complete metadata that will be pushed.
'''
log.info('Enqueing ' + filename)
if compress:
try:
f = File.from_filename_compressed(filename, **metadata_fields)
except OverflowError:
log.warning('Compression failed. Falling back to uncompressed '
'uploads')
f = File.from_filename(filename, **metadata_fields)
else:
f = File.from_filename(filename, **metadata_fields)
fname = f.metadata['id'] + '.tar'
dest = os.path.join(self.queue_dir, fname)
f.to_bundle(dest)
return f
def enqueue(self, filename, compress=False, **metadata_fields):
'''enqueue a file with the specified metadata to be pushed
Args:
filename: the file to enqueue
compress: whether or not to compress the file before enqueueing
Returns the File with complete metadata that will be pushed.
'''
log.info('Enqueing ' + filename)
if compress:
try:
f = File.from_filename_compressed(filename, **metadata_fields)
except OverflowError:
log.warning('Compression failed. Falling back to uncompressed '
'uploads')
f = File.from_filename(filename, **metadata_fields)
else:
f = File.from_filename(filename, **metadata_fields)
fname = f.metadata['id'] + '.tar'
dest = os.path.join(self.queue_dir, fname)
f.to_bundle(dest)
return f
def random_file(tmpdir, metadata=None):
name = random_word(10)
content = random_word(256)
f = tmpdir.join(name)
f.write(content)
if metadata is None:
metadata = generate_random_metadata()
return File.from_filename(f.strpath, **metadata)
def random_file(tmpdir, metadata=None):
name = random_word(10)
content = random_word(256)
f = tmpdir.join(name)
f.write(content)
if metadata is None:
metadata = random_metadata()
return File.from_filename(f.strpath, **metadata)
def prepare_metadata_and_push(self, filename, **metadata_fields):
'''push a file to the archive with the specified metadata
Args:
filename: path of the file to push
metadata_fields: metadata fields for file. Missing fields will be
added if they can be determined. Othwerise, InvalidDatalakeMetadata
will be raised.
returns the url to which the file was pushed.
'''
f = File.from_filename(filename, **metadata_fields)
return self.push(f)
def enqueue(self, filename, compress=False, **metadata_fields):
'''enqueue a file with the specified metadata to be pushed
Args:
filename: the file to enqueue
compress: whether or not to compress the file before enqueueing
Returns the File with complete metadata that will be pushed.
'''
log.info('Enqueing ' + filename)
if compress:
f = File.from_filename_compressed(filename, **metadata_fields)
else:
f = File.from_filename(filename, **metadata_fields)
fname = f.metadata['id'] + '.tar'
dest = os.path.join(self.queue_dir, fname)
f.to_bundle(dest)
return f
def test_non_existent_file():
with pytest.raises(IOError):
File.from_filename('surelythisfiledoesnotexist.txt')
def test_non_existent_file():
with pytest.raises(IOError):
File.from_filename('surelythisfiledoesnotexist.txt')