def test_upload_basic(self):
"""Test a basic minimal upload."""
with BuildSampleData() as sample_files:
md_obj = MetaData([
MetaObj(destinationTable='Transactions._id', value=1234),
MetaObj(destinationTable='Transactions.submitter', value=10),
MetaObj(destinationTable='Transactions.project',
value=u'1234a'),
MetaObj(destinationTable='Transactions.instrument', value=54)
])
bundle_fd = NamedTemporaryFile(delete=False)
bundle = bundler.Bundler(md_obj, sample_files)
bundle.stream(bundle_fd)
bundle_fd.close()
bundle_size = stat(bundle_fd.name).st_size
up_obj = Uploader()
rbundle_fd = open(bundle_fd.name, 'r')
job_id = up_obj.upload(rbundle_fd, content_length=bundle_size)
rbundle_fd.close()
self.assertTrue(job_id)
status = up_obj.getstate(job_id)
def check(status):
"""Check the status since it's complicated."""
check = status['state'] != 'OK'
check |= status['task'] != 'ingest metadata'
check |= int(float(status['task_percent'])) != 100
return check
while check(status):
sleep(1)
status = up_obj.getstate(job_id)
unlink(bundle_fd.name)
self.assertTrue(status)
self.assertEqual(status['state'], 'OK')
def test_bundler_invalid_md_obj(self):
"""Test the bundler create with invalid md object."""
md_obj = MetaData([MetaObj(value=None)])
hit_exception = False
try:
bundler.Bundler(md_obj, [])
except ValueError:
hit_exception = True
self.assertTrue(hit_exception)
def test_bundler_invalid_hash_algo(self):
"""Test the bundler create with invalid hash algo."""
md_obj = MetaData([MetaObj(value='1234')])
hit_exception = False
try:
bundler.Bundler(md_obj, [], hashfunc='blarg')
except ValueError:
hit_exception = True
self.assertTrue(hit_exception)
def test_bundler_basic(self):
"""Test the bundler to stream a tarfile."""
with BuildSampleData() as sample_files:
md_obj = MetaData([MetaObj(value='SomethingReal')])
bundle_fd = NamedTemporaryFile(delete=False)
bundle = bundler.Bundler(md_obj, sample_files)
bundle.stream(bundle_fd)
bundle_fd.close()
self.assertTrue(bundle_fd)
check_tar = TarFile(bundle_fd.name, 'r')
md_fd = check_tar.extractfile('metadata.txt')
self.assertTrue(md_fd)
md_bytes = md_fd.read()
self.assertTrue(loads(md_bytes.decode('utf8')))
def make_bundle():
"""Make the bundler out of files on cmdline."""
upload_files = upload_files_from_args(args.files, args.followlinks,
md_update.directory_prefix())
LOGGER.debug(upload_files)
LOGGER.debug(md_update)
bundle = bundler.Bundler(md_update, upload_files)
bundle.stream(wfd,
callback=lambda x: stdout.write(
'\r' + ' ' * 80 + '\r{:03.2f}%\r'.format(x * 100)),
sleeptime=2)
wfd.close()
stdout.write('\r \r')
LOGGER.debug('Done with making bundle.')
def tar_in_tar(rfd, wfd, md_update, bundle_size):
"""Generate another bundler and wrap rfd in that tar."""
upload_files = [{
'fileobj': rfd,
'name': 'data/data.tar',
'size': bundle_size,
'mtime': (datetime.now() - datetime(1970, 1, 1)).total_seconds()
}]
try:
bundle = bundler.Bundler(md_update, upload_files)
bundle.stream(wfd)
finally:
rfd.close()
wfd.close()
def test_bundler_basic_with_cb(self):
"""Test the bundler to stream a tarfile."""
with BuildSampleData() as sample_files:
md_obj = MetaData([MetaObj(value='SomethingReal')])
bundle_fd = NamedTemporaryFile()
hit_callback = {}
hit_callback['check'] = False
def callback(percent):
"""Callback to set the check."""
hit_callback['check'] = True
self.assertTrue(percent)
bundle = bundler.Bundler(md_obj, sample_files)
bundle.stream(bundle_fd, callback=callback, sleeptime=0)
self.assertTrue(hit_callback)