def test_log_event_error(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelIndicator',
'test_case': test_case_name
})
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name
})
# Create batch owner
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
# Create data quality indicator
indicator = Operation('ModelIndicator').create(
name=test_case_name,
description=test_case_name,
indicatorTypeId=1,
batchOwnerId=batch_owner.id,
executionOrder=0,
active=True)
# Start batch, session and fail session
batch_record = MethodBatch(batch_owner.id).start()
MethodEvent('Start').log_event(indicator.id, batch_record.id)
error_event = MethodEvent('Error').log_event(indicator.id,
batch_record.id)
session_list = Operation('ModelSession').read(indicatorId=indicator.id,
batchId=batch_record.id)
self.assertEqual(session_list[0].statusId, 3)
self.assertEqual(error_event.eventTypeId, 3)
self.assertEqual(error_event.sessionId, session_list[0].id)
def test_delete(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
Operation('ModelBatchOwner').delete(id=batch_owner.id)
batch_owner_list = Operation('ModelBatchOwner').read(
name=test_case_name)
self.assertEqual(batch_owner_list, [])
def test_read(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
Operation('ModelBatchOwner').create(name=test_case_name)
batch_owner_list = Operation('ModelBatchOwner').read(
name=test_case_name)
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name
})
self.assertEqual(batch_owner_list[0].name, test_case_name)
def test_update(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
test_case_name_new = test_utils.get_test_case_name(self.test_case_list)
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
batch_owner = Operation('ModelBatchOwner').update(
id=batch_owner.id, name=test_case_name_new)
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name_new
})
self.assertEqual(batch_owner.name, test_case_name_new)
def test_get_database_connection(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelDataSource',
'test_case': test_case_name
})
# Create data source
current_directory = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_directory = os.path.dirname(current_directory)
Operation('ModelDataSource').create(
name=test_case_name,
dataSourceTypeId=6, # SQLite
connectionString=parent_directory + '/database/data_quality.db',
login='',
password='')
# Get data source connection and execute dummy query
connection = MethodDataSource(test_case_name).get_database_connection()
cursor = connection.cursor()
cursor.execute("select 'Hello World' as text")
for row in cursor:
self.assertEqual(row[0], 'Hello World')
def test_batch_start(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({'class': 'ModelBatchOwner', 'test_case': test_case_name})
# Create batch owner
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
# Start batch
batch_record = MethodBatch(batch_owner.id).start()
self.assertEqual(batch_record.batchOwnerId, batch_owner.id)
self.assertEqual(batch_record.statusId, 1)
def create_db(db_name):
db_path = os.path.join(os.path.dirname(__file__), db_name)
db_uri = 'sqlite:///{}'.format(db_path)
engine = create_engine(db_uri)
# Create tables
log.info('Create database {}'.format(db_name))
Base.metadata.create_all(engine)
# Insert default list of values
with open('api/database/data.json', 'r') as data_file:
data_dictionary = literal_eval(data_file.read())
for object in data_dictionary['list_of_values']:
log.info('Insert default list of values for: {}'.format(object['class']))
for record in object['records']:
try:
Operation(object['class']).create(**record)
except exc.IntegrityError:
log.info('Record already exist: {}'.format(record))
def test_get_data_frame(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelDataSource',
'test_case': test_case_name
})
# Create data source
current_directory = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_directory = os.path.dirname(current_directory)
Operation('ModelDataSource').create(
name=test_case_name,
dataSourceTypeId=6, # SQLite
connectionString=parent_directory + '/database/data_quality.db',
login='',
password='')
# Get data frame
request = "select 'status', count(*) as nb_records from status"
data_frame = MethodDataSource(test_case_name).get_data_frame(request)
for row in data_frame.index:
self.assertGreaterEqual(data_frame.loc[row, 'nb_records'], 0)
def tearDownClass(self):
"""Tear down function called when class is deconstructed."""
for test_case in self.test_case_list:
Operation(test_case['class']).delete(name=test_case['test_case'])
def tearDownClass(self):
"""Tear down function called when class is deconstructed."""
for test_case in self.test_case_list:
Operation('ModelDataSource').delete(name=test_case)
Operation('ModelIndicator').delete(name=test_case)
Operation('ModelBatchOwner').delete(name=test_case)
def setUpClass(self):
"""Set up function called when class is consructed."""
self.base_url = 'http://{}:5000/dataquality/api/graphql'.format(
socket.gethostname())
self.headers = {'content-type': 'application/json'}
self.test_case_name = test_utils.get_test_case_name([])
self.test_case_list = [self.test_case_name]
# Create test data set for subsequent query test cases
# Create data source
data_source_data = {}
data_source_data['name'] = self.test_case_name
data_source_data['dataSourceTypeId'] = 1 # Hive
data_source_data['connectionString'] = self.test_case_name
data_source_data['login'] = self.test_case_name
data_source_data['password'] = self.test_case_name
self.data_source = Operation('ModelDataSource').create(
**data_source_data)
# Create batch owner
batch_owner_data = {}
batch_owner_data['name'] = self.test_case_name
self.batch_owner = Operation('ModelBatchOwner').create(
**batch_owner_data)
# Create indicator
indicator_data = {}
indicator_data['name'] = self.test_case_name
indicator_data['description'] = self.test_case_name
indicator_data['indicatorTypeId'] = 1 # Completeness
indicator_data['batchOwnerId'] = self.batch_owner.id
indicator_data['executionOrder'] = 1
indicator_data['active'] = True
self.indicator = Operation('ModelIndicator').create(**indicator_data)
# Create indicator parameter
parameter_data = {}
parameter_data['indicatorId'] = self.indicator.id
parameter_data['parameterTypeId'] = 6 # Target request
parameter_data['value'] = self.test_case_name
self.parameter = Operation('ModelIndicatorParameter').create(
**parameter_data)
# Create batch
batch_data = {}
batch_data['batchOwnerId'] = self.batch_owner.id
batch_data['statusId'] = 1 # Running
self.batch = Operation('ModelBatch').create(**batch_data)
# Create session
session_data = {}
session_data['indicatorId'] = self.indicator.id
session_data['batchId'] = self.batch.id
session_data['statusId'] = 1 # Running
self.session = Operation('ModelSession').create(**session_data)
# Create event
event_data = {}
event_data['sessionId'] = self.session.id
event_data['eventTypeId'] = 1 # Start
self.event = Operation('ModelEvent').create(**event_data)
# Create indicator result
result_data = {}
result_data['indicatorId'] = self.indicator.id
result_data['sessionId'] = self.session.id
result_data['alertOperator'] = '='
result_data['alertThreshold'] = 0
result_data['nbRecords'] = 0
result_data['nbRecordsAlert'] = 0
result_data['nbRecordsNoAlert'] = 0
self.result = Operation('ModelIndicatorResult').create(**result_data)
def test_execute_freshness(self):
"""Test execute freshness indicator."""
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelIndicator',
'test_case': test_case_name
})
self.test_case_list.append({
'class': 'ModelDataSource',
'test_case': test_case_name
})
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name
})
# Create batch owner
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
# Create data source
current_directory = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_directory = os.path.dirname(current_directory)
data_source = Operation('ModelDataSource').create(
name=test_case_name,
dataSourceTypeId=6, # SQLite
connectionString=parent_directory + '/database/data_quality.db',
login='',
password='')
# Create indicator
indicator_record = Operation('ModelIndicator').create(
name=test_case_name,
description=test_case_name,
indicatorTypeId=2, # Freshness
batchOwnerId=batch_owner.id,
executionOrder=0,
active=1)
# Create indicator paramters
param = Operation('ModelIndicatorParameter')
param.create(parameterTypeId=1,
value=">=",
indicatorId=indicator_record.id) # Alert operator
param.create(parameterTypeId=2,
value="0",
indicatorId=indicator_record.id) # Alert threshold
param.create(parameterTypeId=3,
value="['last_updated_date']",
indicatorId=indicator_record.id) # Measures
param.create(parameterTypeId=4,
value="['table_name']",
indicatorId=indicator_record.id) # Dimensions
param.create(parameterTypeId=5,
value=data_source.name,
indicatorId=indicator_record.id) # Target
param.create(parameterTypeId=6,
value="select 'status', max(updated_date) from status",
indicatorId=indicator_record.id) # Target request
param.create(parameterTypeId=9,
value="['*****@*****.**']",
indicatorId=indicator_record.id) # Distribution list
# Start batch, execute indicator and stop batch
batch_record = MethodBatch(batch_owner.id).start()
MethodIndicator(indicator_record.id).execute(batch_record.id)
MethodBatch(batch_owner.id).stop(batch_record.id)
session = Operation('ModelSession').read(
indicatorId=indicator_record.id, batchId=batch_record.id)
self.assertEqual(session[0].statusId, 2)