def test_db_existence(self):
dh = DataHub.get(self.data_source)
dbs = dh.get_databases()
if 'test' not in dbs:
msg = "No 'test' database found in %s. To run this method create a 'test' db in %s." % (self.data_source, self.data_source)
self.fail(msg)
def __init__(self):
procs_path = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
master_host_config = {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default'].get('PASSWORD') or '',
}
if 'OPTIONS' in settings.DATABASES['default']:
master_host_config.update(settings.DATABASES['default']['OPTIONS'])
read_host_config = {
"host": settings.DATABASES['read_only']['HOST'],
"user": settings.DATABASES['read_only']['USER'],
"passwd": settings.DATABASES['read_only'].get('PASSWORD') or '',
}
if 'OPTIONS' in settings.DATABASES['read_only']:
read_host_config.update(settings.DATABASES['read_only']['OPTIONS'])
data_source = {
'reference': {
"hub": "MySQL",
"master_host": master_host_config,
"read_host": read_host_config,
"require_host_type": True,
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
# Support structure for reference data signatures
self.build_signature_placeholders = []
# Support structures for building option collection data structures
self.oc_hash_lookup = dict()
self.oc_where_in_list = []
self.oc_placeholders = []
self.oc_unique_collections = []
# Support structures for building option data structures
self.o_lookup = set()
self.o_placeholders = []
self.o_unique_options = []
self.o_where_in_list = []
# reference id lookup structure
self.id_lookup = {}
def test_create_data_table(self):
dh = DataHub.get(self.data_source)
dh.execute(db=self.db, proc="test.create_table")
table_set = dh.execute(db=self.db,
proc="sql.ds_selects.get_tables",
key_column="Tables_in_test",
return_type="set")
if self.table_name not in table_set:
msg = "The table, %s, was not created in %s." % (self.table_name,
self.db)
self.fail(msg)
def test_create_data_table(self):
dh = DataHub.get(self.data_source)
dh.execute(db=self.db,
proc="test.create_table")
table_set = dh.execute(db=self.db,
proc="sql.ds_selects.get_tables",
key_column="Tables_in_test",
return_type="set")
if self.table_name not in table_set:
msg = "The table, %s, was not created in %s." % (self.table_name, self.db)
self.fail(msg)
def testDropTable(self):
dh = DataHub.get(self.dataSource)
dh.execute(db=self.db,
proc="test.drop_table")
tableSet = dh.execute(db=self.db,
proc="sql.ds_selects.get_tables",
key_column="Tables_in_test",
return_type="set")
if self.tableName in tableSet:
msg = "The table, %s, was not dropped in %s." % (self.tableName, self.db)
self.fail(msg)
def test_load_data(self):
dh = DataHub.get(self.data_source)
dh.use_database('test')
##Load Data##
for row in TestDataHub.test_data:
dh.execute(proc="test.insert_test_data", placeholders=row)
rowcount = dh.execute(db=self.db,
proc="sql.ds_selects.get_row_count",
replace=['auto_pfamA', self.table_name],
return_type='iter').get_column_data('rowcount')
##Confirm we loaded all of the rows##
msg = 'Row count in data file, %i, does not match row count in db %i.' % (
TestDataHub.test_data_rows, rowcount)
self.assertEqual(rowcount, TestDataHub.test_data_rows, msg=msg)
def test_load_data(self):
dh = DataHub.get(self.data_source)
dh.use_database('test')
# Load Data
for row in TestDataHub.test_data:
dh.execute(proc="test.insert_test_data",
placeholders=row)
rowcount = dh.execute(db=self.db,
proc="sql.ds_selects.get_row_count",
replace=['auto_pfamA', self.table_name],
return_type='iter').get_column_data('rowcount')
# Confirm we loaded all of the rows
msg = 'Row count in data file, %i, does not match row count in db %i.' % (TestDataHub.test_data_rows, rowcount)
self.assertEqual(rowcount, TestDataHub.test_data_rows, msg=msg)
def __init__(self):
procs_path = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
data_source = {
'reference': {
"hub": "MySQL",
"master_host": {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default']['PASSWORD']
},
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
# Support structures for building build platform SQL
self.build_platform_lookup = {}
self.build_where_filters = []
self.build_platform_placeholders = []
self.build_unique_platforms = []
# Support structures for building machine platform SQL
self.machine_platform_lookup = {}
self.machine_where_filters = []
self.machine_platform_placeholders = []
self.machine_unique_platforms = []
# Support structures for building job group SQL
self.job_group_lookup = {}
self.job_group_where_filters = []
self.job_group_placeholders = []
self.job_group_names_and_symbols = []
# Support structures for building job types SQL
self.job_type_lookup = {}
self.job_type_where_filters = []
self.job_type_placeholders = []
self.job_type_names_and_symbols = []
#Use this structure to map the job to the group id
self.job_type_to_group_lookup = {}
# Support structures for building product SQL
self.product_lookup = set()
self.product_where_in_list = []
self.product_placeholders = []
self.unique_products = []
# Support structures for building machine SQL
self.machine_name_lookup = set()
self.machine_where_in_list = []
self.machine_name_placeholders = []
self.machine_unique_names = []
self.machine_timestamp_update_placeholders = []
# Support structures for building option collection data structures
self.oc_hash_lookup = dict()
self.oc_where_in_list = []
self.oc_placeholders = []
self.oc_unique_collections = []
# Support structures for building option data structures
self.o_lookup = set()
self.o_placeholders = []
self.o_unique_options = []
self.o_where_in_list = []
# reference id lookup structure
self.id_lookup = {}
def __init__(self):
procs_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
data_source = {
'reference': {
"hub": "MySQL",
"master_host": {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default']['PASSWORD']
},
"read_host": {
"host": settings.DATABASES['read_only']['HOST'],
"user": settings.DATABASES['read_only']['USER'],
"passwd": settings.DATABASES['read_only']['PASSWORD']
},
"require_host_type": True,
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
# Support structure for reference data signatures
self.reference_data_signature_lookup = {}
self.build_signature_placeholders = []
# Support structures for building build platform SQL
self.build_platform_lookup = {}
self.build_where_filters = []
self.build_platform_placeholders = []
self.build_unique_platforms = []
# Support structures for building machine platform SQL
self.machine_platform_lookup = {}
self.machine_where_filters = []
self.machine_platform_placeholders = []
self.machine_unique_platforms = []
# Support structures for building job group SQL
self.job_group_lookup = {}
self.job_group_where_filters = []
self.job_group_placeholders = []
self.job_group_names_and_symbols = []
# Support structures for building job types SQL
self.job_type_lookup = {}
self.job_type_where_filters = []
self.job_type_placeholders = []
self.job_type_names_and_symbols = []
# Use this structure to map the job to the group id
self.job_type_to_group_lookup = {}
# Support structures for building product SQL
self.product_lookup = set()
self.product_where_in_list = []
self.product_placeholders = []
self.unique_products = []
# Support structures for building device SQL
self.device_lookup = set()
self.device_where_in_list = []
self.device_placeholders = []
self.unique_devices = []
# Support structures for building machine SQL
self.machine_name_lookup = set()
self.machine_where_in_list = []
self.machine_name_placeholders = []
self.machine_unique_names = []
self.machine_timestamp_update_placeholders = []
# Support structures for building option collection data structures
self.oc_hash_lookup = dict()
self.oc_where_in_list = []
self.oc_placeholders = []
self.oc_unique_collections = []
# Support structures for building option data structures
self.o_lookup = set()
self.o_placeholders = []
self.o_unique_options = []
self.o_where_in_list = []
# reference id lookup structure
self.id_lookup = {}
