def test_update(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
test_case_name_new = test_utils.get_test_case_name(self.test_case_list)
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
batch_owner = Operation('ModelBatchOwner').update(
id=batch_owner.id, name=test_case_name_new)
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name_new
})
self.assertEqual(batch_owner.name, test_case_name_new)
def test_log_event_error(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelIndicator',
'test_case': test_case_name
})
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name
})
# Create batch owner
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
# Create data quality indicator
indicator = Operation('ModelIndicator').create(
name=test_case_name,
description=test_case_name,
indicatorTypeId=1,
batchOwnerId=batch_owner.id,
executionOrder=0,
active=True)
# Start batch, session and fail session
batch_record = MethodBatch(batch_owner.id).start()
MethodEvent('Start').log_event(indicator.id, batch_record.id)
error_event = MethodEvent('Error').log_event(indicator.id,
batch_record.id)
session_list = Operation('ModelSession').read(indicatorId=indicator.id,
batchId=batch_record.id)
self.assertEqual(session_list[0].statusId, 3)
self.assertEqual(error_event.eventTypeId, 3)
self.assertEqual(error_event.sessionId, session_list[0].id)
def test_mutation_create_indicator_parameter(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelIndicator", "test_case": test_case_name})
self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name})
# Create batch owner
payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {id}}}"}' % test_case_name
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
batch_owner_id = json['data']['createBatchOwner']['batchOwner']['id']
# Create indicator
indicator_type_id = to_global_id('SchemaIndicatorType', 1) # Completeness
input = '{name:\\"%s\\", description:\\"%s\\", indicatorTypeId:\\"%s\\", batchOwnerId:\\"%s\\", executionOrder: 1, active: true}' % (test_case_name, test_case_name, indicator_type_id, batch_owner_id)
payload = '{"query": "mutation {createIndicator (input: %s) {indicator {id}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
indicator_id = json['data']['createIndicator']['indicator']['id']
# Create indicator parameter
indicator_parameter_type_id = to_global_id('SchemaIndicatorParameterType', 1) # Alert operator
input = '{indicatorId:\\"%s\\", parameterTypeId:\\"%s\\", value:\\"%s\\"}' % (indicator_id, indicator_parameter_type_id, test_case_name)
payload = '{"query": "mutation {createIndicatorParameter (input: %s) {parameter {value parameterType {name}}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
parameter_value = json['data']['createIndicatorParameter']['parameter']['value']
parameter_type_name = json['data']['createIndicatorParameter']['parameter']['parameterType']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(parameter_value, test_case_name)
self.assertEqual(parameter_type_name, 'Alert operator')
def test_get_database_connection(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelDataSource',
'test_case': test_case_name
})
# Create data source
current_directory = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_directory = os.path.dirname(current_directory)
Operation('ModelDataSource').create(
name=test_case_name,
dataSourceTypeId=6, # SQLite
connectionString=parent_directory + '/database/data_quality.db',
login='',
password='')
# Get data source connection and execute dummy query
connection = MethodDataSource(test_case_name).get_database_connection()
cursor = connection.cursor()
cursor.execute("select 'Hello World' as text")
for row in cursor:
self.assertEqual(row[0], 'Hello World')
def test_delete(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
Operation('ModelBatchOwner').delete(id=batch_owner.id)
batch_owner_list = Operation('ModelBatchOwner').read(
name=test_case_name)
self.assertEqual(batch_owner_list, [])
def test_read(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
Operation('ModelBatchOwner').create(name=test_case_name)
batch_owner_list = Operation('ModelBatchOwner').read(
name=test_case_name)
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name
})
self.assertEqual(batch_owner_list[0].name, test_case_name)
def test_mutation_create_event_type(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelEventType", "test_case": test_case_name})
# Create event type
payload = '{"query": "mutation {createEventType (input: {name:\\"%s\\"}) {eventType {name}}}"}' % test_case_name
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
event_type_name = json['data']['createEventType']['eventType']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(event_type_name, test_case_name)
def test_mutation_create_batch_owner(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name})
# Create batch owner
payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {name}}}"}' % test_case_name
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
batch_owner_name = json['data']['createBatchOwner']['batchOwner']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(batch_owner_name, test_case_name)
def test_mutation_create_status(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelStatus", "test_case": test_case_name})
# Create status
payload = '{"query": "mutation {createStatus (input: {name:\\"%s\\"}) {status {name}}}"}' % test_case_name
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
status_name = json['data']['createStatus']['status']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(status_name, test_case_name)
def test_batch_start(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({'class': 'ModelBatchOwner', 'test_case': test_case_name})
# Create batch owner
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
# Start batch
batch_record = MethodBatch(batch_owner.id).start()
self.assertEqual(batch_record.batchOwnerId, batch_owner.id)
self.assertEqual(batch_record.statusId, 1)
def test_mutation_create_indicator_type(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelIndicatorType", "test_case": test_case_name})
# Create indicator type
input = '{name:\\"%s\\", function:\\"%s\\"}' % (test_case_name, test_case_name)
payload = '{"query": "mutation {createIndicatorType (input: %s) {indicatorType {name}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
indicator_type_name = json['data']['createIndicatorType']['indicatorType']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(indicator_type_name, test_case_name)
def test_mutation_create_data_source_type(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelDataSourceType", "test_case": test_case_name})
# Create data source type
input = '{name:\\"%s\\", parentType: \\"%s\\"}' % (test_case_name, test_case_name)
payload = '{"query": "mutation {createDataSourceType (input: %s) {dataSourceType {name}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
data_source_type_name = json['data']['createDataSourceType']['dataSourceType']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(data_source_type_name, test_case_name)
def test_mutation_create_data_source(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelDataSource", "test_case": test_case_name})
# Create data source
data_source_type_id = to_global_id('SchemaDataSourceType', 1) # Hive
input = '{name:\\"%s\\", dataSourceTypeId: \\"%s\\", connectionString: \\"%s\\", login: \\"%s\\", password: \\"%s\\"}' % (test_case_name, data_source_type_id, test_case_name, test_case_name, test_case_name)
payload = '{"query": "mutation {createDataSource (input: %s) {dataSource {name}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
data_source_name = json['data']['createDataSource']['dataSource']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(data_source_name, test_case_name)
def test_mutation_create_indicator_result(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelIndicator", "test_case": test_case_name})
self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name})
# Create batch owner
payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {id}}}"}' % test_case_name
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
batch_owner_id = json['data']['createBatchOwner']['batchOwner']['id']
# Create indicator
indicator_type_id = to_global_id('SchemaIndicatorType', 1) # Completeness
input = '{name:\\"%s\\", description:\\"%s\\", indicatorTypeId:\\"%s\\", batchOwnerId:\\"%s\\", executionOrder: 1, active: true}' % (test_case_name, test_case_name, indicator_type_id, batch_owner_id)
payload = '{"query": "mutation {createIndicator (input: %s) {indicator {id}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
indicator_id = json['data']['createIndicator']['indicator']['id']
# Create batch
status_id = to_global_id('SchemaStatus', 1) # Running
input = '{batchOwnerId:\\"%s\\", statusId: \\"%s\\"}' % (batch_owner_id, status_id)
payload = '{"query": "mutation {createBatch (input: %s) {batch {id}}}"}' % (input)
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
batch_id = json['data']['createBatch']['batch']['id']
# Create session
status_id = to_global_id('SchemaStatus', 1) # Running
input = '{indicatorId:\\"%s\\", batchId: \\"%s\\", statusId: \\"%s\\"}' % (indicator_id, batch_id, status_id)
payload = '{"query": "mutation {createSession (input: %s) {session {id}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
session_id = json['data']['createSession']['session']['id']
# Create indicator result
input = '{indicatorId:\\"%s\\", sessionId:\\"%s\\", alertOperator:\\"=\\", alertThreshold:0.0, nbRecords:0, nbRecordsAlert:0, nbRecordsNoAlert:0}' % (indicator_id, session_id)
payload = '{"query": "mutation {createIndicatorResult (input: %s) {indicatorResult {indicator {name} session {batch {batchOwner {name}}}}}}"}' % input
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
indicator_name = json['data']['createIndicatorResult']['indicatorResult']['indicator']['name']
batch_owner_name = json['data']['createIndicatorResult']['indicatorResult']['session']['batch']['batchOwner']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(indicator_name, test_case_name)
self.assertEqual(batch_owner_name, test_case_name)
def test_mutation_create_batch(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name})
# Create batch owner
payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {id}}}"}' % test_case_name
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
batch_owner_id = json['data']['createBatchOwner']['batchOwner']['id']
# Create batch
status_id = to_global_id('SchemaStatus', 1) # Running
input = '{batchOwnerId:\\"%s\\", statusId: \\"%s\\"}' % (batch_owner_id, status_id)
payload = '{"query": "mutation {createBatch (input: %s) {batch {batchOwner {name} status {name}}}}"}' % (input)
response = requests.post(self.base_url, headers=self.headers, data=payload)
json = response.json()
batch_owner_name = json['data']['createBatch']['batch']['batchOwner']['name']
status_name = json['data']['createBatch']['batch']['status']['name']
self.assertEqual(response.status_code, 200)
self.assertEqual(batch_owner_name, test_case_name)
self.assertEqual(status_name, 'Running')
def test_get_data_frame(self):
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelDataSource',
'test_case': test_case_name
})
# Create data source
current_directory = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_directory = os.path.dirname(current_directory)
Operation('ModelDataSource').create(
name=test_case_name,
dataSourceTypeId=6, # SQLite
connectionString=parent_directory + '/database/data_quality.db',
login='',
password='')
# Get data frame
request = "select 'status', count(*) as nb_records from status"
data_frame = MethodDataSource(test_case_name).get_data_frame(request)
for row in data_frame.index:
self.assertGreaterEqual(data_frame.loc[row, 'nb_records'], 0)
def test_execute_freshness(self):
"""Test execute freshness indicator."""
test_case_name = test_utils.get_test_case_name(self.test_case_list)
self.test_case_list.append({
'class': 'ModelIndicator',
'test_case': test_case_name
})
self.test_case_list.append({
'class': 'ModelDataSource',
'test_case': test_case_name
})
self.test_case_list.append({
'class': 'ModelBatchOwner',
'test_case': test_case_name
})
# Create batch owner
batch_owner = Operation('ModelBatchOwner').create(name=test_case_name)
# Create data source
current_directory = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_directory = os.path.dirname(current_directory)
data_source = Operation('ModelDataSource').create(
name=test_case_name,
dataSourceTypeId=6, # SQLite
connectionString=parent_directory + '/database/data_quality.db',
login='',
password='')
# Create indicator
indicator_record = Operation('ModelIndicator').create(
name=test_case_name,
description=test_case_name,
indicatorTypeId=2, # Freshness
batchOwnerId=batch_owner.id,
executionOrder=0,
active=1)
# Create indicator paramters
param = Operation('ModelIndicatorParameter')
param.create(parameterTypeId=1,
value=">=",
indicatorId=indicator_record.id) # Alert operator
param.create(parameterTypeId=2,
value="0",
indicatorId=indicator_record.id) # Alert threshold
param.create(parameterTypeId=3,
value="['last_updated_date']",
indicatorId=indicator_record.id) # Measures
param.create(parameterTypeId=4,
value="['table_name']",
indicatorId=indicator_record.id) # Dimensions
param.create(parameterTypeId=5,
value=data_source.name,
indicatorId=indicator_record.id) # Target
param.create(parameterTypeId=6,
value="select 'status', max(updated_date) from status",
indicatorId=indicator_record.id) # Target request
param.create(parameterTypeId=9,
value="['*****@*****.**']",
indicatorId=indicator_record.id) # Distribution list
# Start batch, execute indicator and stop batch
batch_record = MethodBatch(batch_owner.id).start()
MethodIndicator(indicator_record.id).execute(batch_record.id)
MethodBatch(batch_owner.id).stop(batch_record.id)
session = Operation('ModelSession').read(
indicatorId=indicator_record.id, batchId=batch_record.id)
self.assertEqual(session[0].statusId, 2)
def setUpClass(self):
"""Set up function called when class is consructed."""
self.base_url = 'http://{}:5000/dataquality/api/graphql'.format(
socket.gethostname())
self.headers = {'content-type': 'application/json'}
self.test_case_name = test_utils.get_test_case_name([])
self.test_case_list = [self.test_case_name]
# Create test data set for subsequent query test cases
# Create data source
data_source_data = {}
data_source_data['name'] = self.test_case_name
data_source_data['dataSourceTypeId'] = 1 # Hive
data_source_data['connectionString'] = self.test_case_name
data_source_data['login'] = self.test_case_name
data_source_data['password'] = self.test_case_name
self.data_source = Operation('ModelDataSource').create(
**data_source_data)
# Create batch owner
batch_owner_data = {}
batch_owner_data['name'] = self.test_case_name
self.batch_owner = Operation('ModelBatchOwner').create(
**batch_owner_data)
# Create indicator
indicator_data = {}
indicator_data['name'] = self.test_case_name
indicator_data['description'] = self.test_case_name
indicator_data['indicatorTypeId'] = 1 # Completeness
indicator_data['batchOwnerId'] = self.batch_owner.id
indicator_data['executionOrder'] = 1
indicator_data['active'] = True
self.indicator = Operation('ModelIndicator').create(**indicator_data)
# Create indicator parameter
parameter_data = {}
parameter_data['indicatorId'] = self.indicator.id
parameter_data['parameterTypeId'] = 6 # Target request
parameter_data['value'] = self.test_case_name
self.parameter = Operation('ModelIndicatorParameter').create(
**parameter_data)
# Create batch
batch_data = {}
batch_data['batchOwnerId'] = self.batch_owner.id
batch_data['statusId'] = 1 # Running
self.batch = Operation('ModelBatch').create(**batch_data)
# Create session
session_data = {}
session_data['indicatorId'] = self.indicator.id
session_data['batchId'] = self.batch.id
session_data['statusId'] = 1 # Running
self.session = Operation('ModelSession').create(**session_data)
# Create event
event_data = {}
event_data['sessionId'] = self.session.id
event_data['eventTypeId'] = 1 # Start
self.event = Operation('ModelEvent').create(**event_data)
# Create indicator result
result_data = {}
result_data['indicatorId'] = self.indicator.id
result_data['sessionId'] = self.session.id
result_data['alertOperator'] = '='
result_data['alertThreshold'] = 0
result_data['nbRecords'] = 0
result_data['nbRecordsAlert'] = 0
result_data['nbRecordsNoAlert'] = 0
self.result = Operation('ModelIndicatorResult').create(**result_data)
