def test_md_extend(self):
"""Test the append features of MetaData."""
md_obj = MetaData([MetaObj(destinationTable='blarg')])
md_obj.extend(
MetaData([MetaObj(destinationTable='blarg2', metaID='aboodaba')]))
self.assertTrue(len(md_obj) == 2)
self.assertTrue(md_obj[1] == md_obj['aboodaba'])
def test_md_insert(self):
"""Test the insert features of MetaData."""
md_obj = MetaData([
MetaObj(destinationTable='blarg'),
MetaObj(destinationTable='blarg2', metaID='aboodaba')
])
mo_obj = MetaObj(destinationTable='insert', metaID='inserted')
md_obj.insert(1, mo_obj)
self.assertTrue(md_obj[1] == mo_obj)
self.assertTrue(md_obj['inserted'] == mo_obj)
def test_md_remove(self):
"""Test the delete features of MetaData."""
mo_obj = MetaObj(destinationTable='blarg', metaID='aboodaba')
md_obj = MetaData([mo_obj])
md_obj.remove(mo_obj)
self.assertFalse(md_obj)
hit_exception = False
try:
md_obj['aboodaba']
except IndexError:
hit_exception = True
self.assertTrue(hit_exception)
def test_md_pop(self):
"""Test the pop features of MetaData."""
md_obj = MetaData([
MetaObj(destinationTable='blarg'),
MetaObj(destinationTable='blarg2', metaID='aboodaba')
])
self.assertTrue(len(md_obj) == 2)
mo_obj = md_obj.pop()
self.assertTrue(mo_obj.metaID == 'aboodaba')
hit_exception = False
try:
md_obj['abodaba']
except IndexError:
hit_exception = True
self.assertTrue(hit_exception)
def _to_meta_data(
transaction: Transaction,
transaction_key_values: typing.List[TransactionKeyValue]) -> MetaData:
meta_objs = []
if transaction is not None:
for name in ['_id']:
value = getattr(transaction, name, None)
if value is not None:
raise TransactionDuplicateAttributeError(None, name)
for name in [
'analytical_tool', 'description', 'instrument', 'project',
'submitter', 'suspense_date'
]:
value = getattr(transaction, name, None)
if value is not None:
meta_obj = MetaObj(
destinationTable='Transactions.{0}'.format(name),
value=value)
meta_objs.append(meta_obj)
for transaction_key_value in transaction_key_values:
meta_obj = MetaObj(destinationTable='TransactionKeyValue',
key=transaction_key_value.key,
value=transaction_key_value.value)
meta_objs.append(meta_obj)
meta_data = MetaData(meta_objs)
return meta_data
def test_upload_basic(self):
"""Test a basic minimal upload."""
with BuildSampleData() as sample_files:
md_obj = MetaData([
MetaObj(destinationTable='Transactions._id', value=1234),
MetaObj(destinationTable='Transactions.submitter', value=10),
MetaObj(destinationTable='Transactions.project',
value=u'1234a'),
MetaObj(destinationTable='Transactions.instrument', value=54)
])
bundle_fd = NamedTemporaryFile(delete=False)
bundle = bundler.Bundler(md_obj, sample_files)
bundle.stream(bundle_fd)
bundle_fd.close()
bundle_size = stat(bundle_fd.name).st_size
up_obj = Uploader()
rbundle_fd = open(bundle_fd.name, 'r')
job_id = up_obj.upload(rbundle_fd, content_length=bundle_size)
rbundle_fd.close()
self.assertTrue(job_id)
status = up_obj.getstate(job_id)
def check(status):
"""Check the status since it's complicated."""
check = status['state'] != 'OK'
check |= status['task'] != 'ingest metadata'
check |= int(float(status['task_percent'])) != 100
return check
while check(status):
sleep(1)
status = up_obj.getstate(job_id)
unlink(bundle_fd.name)
self.assertTrue(status)
self.assertEqual(status['state'], 'OK')
def test_error_mo_encoding(self):
"""Fail to parse a MetaData object."""
hit_exception = False
try:
metadata_encode(MetaData([complex('1+2j')]))
except TypeError:
hit_exception = True
self.assertTrue(hit_exception)
def test_bundler_invalid_md_obj(self):
"""Test the bundler create with invalid md object."""
md_obj = MetaData([MetaObj(value=None)])
hit_exception = False
try:
bundler.Bundler(md_obj, [])
except ValueError:
hit_exception = True
self.assertTrue(hit_exception)
def test_bundler_invalid_hash_algo(self):
"""Test the bundler create with invalid hash algo."""
md_obj = MetaData([MetaObj(value='1234')])
hit_exception = False
try:
bundler.Bundler(md_obj, [], hashfunc='blarg')
except ValueError:
hit_exception = True
self.assertTrue(hit_exception)
def test_encoding_plus_extras(self):
"""Test the metadata encoding with simple example."""
md_obj = MetaData([MetaObj(destinationTable='blarg'), {'foo': 'bar'}])
hit_exception = False
try:
metadata_encode(md_obj)
except TypeError:
hit_exception = True
self.assertTrue(hit_exception)
def test_md_setitem_error(self):
"""A FileObj can't be inserted by an ID it doesn't have."""
md_obj = MetaData([
MetaObj(destinationTable='blarg'),
MetaObj(destinationTable='blarg2', metaID='aboodaba')
])
file_obj = FileObj()
hit_exception = False
try:
md_obj['aboodaba'] = file_obj
except IndexError:
hit_exception = True
self.assertTrue(hit_exception)
def test_bundler_basic(self):
"""Test the bundler to stream a tarfile."""
with BuildSampleData() as sample_files:
md_obj = MetaData([MetaObj(value='SomethingReal')])
bundle_fd = NamedTemporaryFile(delete=False)
bundle = bundler.Bundler(md_obj, sample_files)
bundle.stream(bundle_fd)
bundle_fd.close()
self.assertTrue(bundle_fd)
check_tar = TarFile(bundle_fd.name, 'r')
md_fd = check_tar.extractfile('metadata.txt')
self.assertTrue(md_fd)
md_bytes = md_fd.read()
self.assertTrue(loads(md_bytes.decode('utf8')))
def test_md_setitem(self):
"""Test the delete features of MetaData."""
mo_obj = MetaObj(destinationTable='blarg', metaID='aboodaba')
md_obj = MetaData([MetaObj(destinationTable='blah', metaID='blarg')])
md_obj[0] = mo_obj
self.assertTrue(len(md_obj) == 1)
self.assertTrue(md_obj[0] == mo_obj)
self.assertTrue(md_obj['aboodaba'] == mo_obj)
hit_exception = False
try:
md_obj['blarg']
except IndexError:
hit_exception = True
self.assertTrue(hit_exception)
def test_bundler_basic_with_cb(self):
"""Test the bundler to stream a tarfile."""
with BuildSampleData() as sample_files:
md_obj = MetaData([MetaObj(value='SomethingReal')])
bundle_fd = NamedTemporaryFile()
hit_callback = {}
hit_callback['check'] = False
def callback(percent):
"""Callback to set the check."""
hit_callback['check'] = True
self.assertTrue(percent)
bundle = bundler.Bundler(md_obj, sample_files)
bundle.stream(bundle_fd, callback=callback, sleeptime=0)
self.assertTrue(hit_callback)
def test_encoding_with_files(self):
"""Test the metadata encoding with simple example."""
md_obj = MetaData([FileObj(destinationTable='Files')])
meta_str = metadata_encode(md_obj)
self.assertTrue(meta_str == '[{"destinationTable": "Files"}]')
def test_encoding(self):
"""Test the metadata encoding with simple example."""
md_obj = MetaData([MetaObj(destinationTable='blarg')])
meta_str = metadata_encode(md_obj)
self.assertTrue(meta_str == '[{"destinationTable": "blarg"}]')