def persist_bundle(self):
from madmex.persistence.driver import persist_bundle
from sqlalchemy import create_engine
from sqlalchemy.orm.session import sessionmaker
from madmex.util import remove_file
dummy = DummyBundle()
persist_bundle(dummy)
my_database = getattr(SETTINGS, 'ANTARES_TEST_DATABASE')
klass = sessionmaker(bind=create_engine(my_database))
session = klass()
query = 'SELECT count(*) FROM product WHERE uuid=\'%s\';' % dummy.uuid_id
print query
try:
result_set = session.execute(query)
for row in result_set:
self.assertGreater(row['count'], 0)
# Delete object from database.
session.delete(dummy.get_database_object())
session.commit()
for file_name in dummy.get_files():
full_path = os.path.join(dummy.get_output_directory(), os.path.basename(file_name))
self.assertTrue(os.path.isfile(full_path))
# Remove file from filesystem.
remove_file(full_path)
except:
session.rollback()
raise
finally:
session.close()
def persist_bundle_sensor(self):
from madmex.persistence.driver import persist_bundle
folder = '/LUSTRE/MADMEX/staging/madmex_antares/test_ingest/556_297_041114_dim_img_spot'
from sqlalchemy import create_engine
from sqlalchemy.orm.session import sessionmaker
from madmex.mapper.bundle.spot5 import Bundle
#from madmex.configuration import SETTINGS
dummy = Bundle(folder)
#dummy.target = '/LUSTRE/MADMEX/staging/'
target_url = getattr(SETTINGS, 'TEST_FOLDER')
print target_url
#TODO please fix me, horrible hack
dummy.target = target_url
persist_bundle(dummy)
my_database = getattr(SETTINGS, 'ANTARES_TEST_DATABASE')
klass = sessionmaker(bind=create_engine(my_database))
session = klass()
query = 'SELECT count(*) FROM product WHERE uuid=\'%s\';' % dummy.uuid_id
try:
result_set = session.execute(query)
for row in result_set:
self.assertGreater(row['count'], 0)
session.delete(dummy.get_database_object())
session.commit()
for file_name in dummy.get_files():
full_path = os.path.join(target_url, os.path.basename(file_name))
self.assertTrue(os.path.isfile(full_path))
os.remove(full_path)
except:
session.rollback()
raise
finally:
session.close()
def handle(self, **options):
'''
This is the code that does the ingestion.
'''
keep = options['keep']
for path in options['path']:
bundle = _get_bundle_from_path(path)
if bundle:
LOGGER.info('Directory %s is a %s bundle.', path, bundle.get_name())
persist_bundle(bundle, keep)
else:
LOGGER.info('No bundle was able to identify the directory: %s.', path)
def handle(self, **options):
'''
This is the code that does the ingestion.
'''
keep = options['keep']
for path in options['path']:
bundle = _get_bundle_from_path(path)
if bundle:
LOGGER.info('Directory %s is a %s bundle.', path,
bundle.get_name())
persist_bundle(bundle, keep)
else:
LOGGER.info(
'No bundle was able to identify the directory: %s.', path)
def test_persist_bundle_with_error(self):
'''
Tests the behavior of persisting a bundle object when the file to
be persisted in the filesystem does not exists.
'''
from madmex.persistence.driver import persist_bundle
dummy = ErrorDummyBundle()
persist_bundle(dummy)
query = 'SELECT count(*) FROM product WHERE uuid=\'%s\';' % dummy.uuid_id
try:
result_set = self.get_session().execute(query)
for row in result_set:
print dir(row.keys)
print row.keys
self.assertEqual(row['count'], 0)
for file_name in dummy.get_files():
full_path = os.path.join(dummy.get_output_directory(), os.path.basename(file_name))
self.assertFalse(os.path.isfile(full_path))
except:
self.get_session().rollback()
raise
finally:
self.get_session().close()
