def test_update(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) test_case_name_new = test_utils.get_test_case_name(self.test_case_list) batch_owner = Operation('ModelBatchOwner').create(name=test_case_name) batch_owner = Operation('ModelBatchOwner').update( id=batch_owner.id, name=test_case_name_new) self.test_case_list.append({ 'class': 'ModelBatchOwner', 'test_case': test_case_name_new }) self.assertEqual(batch_owner.name, test_case_name_new)
def test_log_event_error(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({ 'class': 'ModelIndicator', 'test_case': test_case_name }) self.test_case_list.append({ 'class': 'ModelBatchOwner', 'test_case': test_case_name }) # Create batch owner batch_owner = Operation('ModelBatchOwner').create(name=test_case_name) # Create data quality indicator indicator = Operation('ModelIndicator').create( name=test_case_name, description=test_case_name, indicatorTypeId=1, batchOwnerId=batch_owner.id, executionOrder=0, active=True) # Start batch, session and fail session batch_record = MethodBatch(batch_owner.id).start() MethodEvent('Start').log_event(indicator.id, batch_record.id) error_event = MethodEvent('Error').log_event(indicator.id, batch_record.id) session_list = Operation('ModelSession').read(indicatorId=indicator.id, batchId=batch_record.id) self.assertEqual(session_list[0].statusId, 3) self.assertEqual(error_event.eventTypeId, 3) self.assertEqual(error_event.sessionId, session_list[0].id)
def test_mutation_create_indicator_parameter(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelIndicator", "test_case": test_case_name}) self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name}) # Create batch owner payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {id}}}"}' % test_case_name response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() batch_owner_id = json['data']['createBatchOwner']['batchOwner']['id'] # Create indicator indicator_type_id = to_global_id('SchemaIndicatorType', 1) # Completeness input = '{name:\\"%s\\", description:\\"%s\\", indicatorTypeId:\\"%s\\", batchOwnerId:\\"%s\\", executionOrder: 1, active: true}' % (test_case_name, test_case_name, indicator_type_id, batch_owner_id) payload = '{"query": "mutation {createIndicator (input: %s) {indicator {id}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() indicator_id = json['data']['createIndicator']['indicator']['id'] # Create indicator parameter indicator_parameter_type_id = to_global_id('SchemaIndicatorParameterType', 1) # Alert operator input = '{indicatorId:\\"%s\\", parameterTypeId:\\"%s\\", value:\\"%s\\"}' % (indicator_id, indicator_parameter_type_id, test_case_name) payload = '{"query": "mutation {createIndicatorParameter (input: %s) {parameter {value parameterType {name}}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() parameter_value = json['data']['createIndicatorParameter']['parameter']['value'] parameter_type_name = json['data']['createIndicatorParameter']['parameter']['parameterType']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(parameter_value, test_case_name) self.assertEqual(parameter_type_name, 'Alert operator')
def test_get_database_connection(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({ 'class': 'ModelDataSource', 'test_case': test_case_name }) # Create data source current_directory = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) parent_directory = os.path.dirname(current_directory) Operation('ModelDataSource').create( name=test_case_name, dataSourceTypeId=6, # SQLite connectionString=parent_directory + '/database/data_quality.db', login='', password='') # Get data source connection and execute dummy query connection = MethodDataSource(test_case_name).get_database_connection() cursor = connection.cursor() cursor.execute("select 'Hello World' as text") for row in cursor: self.assertEqual(row[0], 'Hello World')
def test_delete(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) batch_owner = Operation('ModelBatchOwner').create(name=test_case_name) Operation('ModelBatchOwner').delete(id=batch_owner.id) batch_owner_list = Operation('ModelBatchOwner').read( name=test_case_name) self.assertEqual(batch_owner_list, [])
def test_read(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) Operation('ModelBatchOwner').create(name=test_case_name) batch_owner_list = Operation('ModelBatchOwner').read( name=test_case_name) self.test_case_list.append({ 'class': 'ModelBatchOwner', 'test_case': test_case_name }) self.assertEqual(batch_owner_list[0].name, test_case_name)
def test_mutation_create_event_type(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelEventType", "test_case": test_case_name}) # Create event type payload = '{"query": "mutation {createEventType (input: {name:\\"%s\\"}) {eventType {name}}}"}' % test_case_name response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() event_type_name = json['data']['createEventType']['eventType']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(event_type_name, test_case_name)
def test_mutation_create_batch_owner(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name}) # Create batch owner payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {name}}}"}' % test_case_name response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() batch_owner_name = json['data']['createBatchOwner']['batchOwner']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(batch_owner_name, test_case_name)
def test_mutation_create_status(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelStatus", "test_case": test_case_name}) # Create status payload = '{"query": "mutation {createStatus (input: {name:\\"%s\\"}) {status {name}}}"}' % test_case_name response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() status_name = json['data']['createStatus']['status']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(status_name, test_case_name)
def test_batch_start(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({'class': 'ModelBatchOwner', 'test_case': test_case_name}) # Create batch owner batch_owner = Operation('ModelBatchOwner').create(name=test_case_name) # Start batch batch_record = MethodBatch(batch_owner.id).start() self.assertEqual(batch_record.batchOwnerId, batch_owner.id) self.assertEqual(batch_record.statusId, 1)
def test_mutation_create_indicator_type(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelIndicatorType", "test_case": test_case_name}) # Create indicator type input = '{name:\\"%s\\", function:\\"%s\\"}' % (test_case_name, test_case_name) payload = '{"query": "mutation {createIndicatorType (input: %s) {indicatorType {name}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() indicator_type_name = json['data']['createIndicatorType']['indicatorType']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(indicator_type_name, test_case_name)
def test_mutation_create_data_source_type(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelDataSourceType", "test_case": test_case_name}) # Create data source type input = '{name:\\"%s\\", parentType: \\"%s\\"}' % (test_case_name, test_case_name) payload = '{"query": "mutation {createDataSourceType (input: %s) {dataSourceType {name}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() data_source_type_name = json['data']['createDataSourceType']['dataSourceType']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(data_source_type_name, test_case_name)
def test_mutation_create_data_source(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelDataSource", "test_case": test_case_name}) # Create data source data_source_type_id = to_global_id('SchemaDataSourceType', 1) # Hive input = '{name:\\"%s\\", dataSourceTypeId: \\"%s\\", connectionString: \\"%s\\", login: \\"%s\\", password: \\"%s\\"}' % (test_case_name, data_source_type_id, test_case_name, test_case_name, test_case_name) payload = '{"query": "mutation {createDataSource (input: %s) {dataSource {name}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() data_source_name = json['data']['createDataSource']['dataSource']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(data_source_name, test_case_name)
def test_mutation_create_indicator_result(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelIndicator", "test_case": test_case_name}) self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name}) # Create batch owner payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {id}}}"}' % test_case_name response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() batch_owner_id = json['data']['createBatchOwner']['batchOwner']['id'] # Create indicator indicator_type_id = to_global_id('SchemaIndicatorType', 1) # Completeness input = '{name:\\"%s\\", description:\\"%s\\", indicatorTypeId:\\"%s\\", batchOwnerId:\\"%s\\", executionOrder: 1, active: true}' % (test_case_name, test_case_name, indicator_type_id, batch_owner_id) payload = '{"query": "mutation {createIndicator (input: %s) {indicator {id}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() indicator_id = json['data']['createIndicator']['indicator']['id'] # Create batch status_id = to_global_id('SchemaStatus', 1) # Running input = '{batchOwnerId:\\"%s\\", statusId: \\"%s\\"}' % (batch_owner_id, status_id) payload = '{"query": "mutation {createBatch (input: %s) {batch {id}}}"}' % (input) response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() batch_id = json['data']['createBatch']['batch']['id'] # Create session status_id = to_global_id('SchemaStatus', 1) # Running input = '{indicatorId:\\"%s\\", batchId: \\"%s\\", statusId: \\"%s\\"}' % (indicator_id, batch_id, status_id) payload = '{"query": "mutation {createSession (input: %s) {session {id}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() session_id = json['data']['createSession']['session']['id'] # Create indicator result input = '{indicatorId:\\"%s\\", sessionId:\\"%s\\", alertOperator:\\"=\\", alertThreshold:0.0, nbRecords:0, nbRecordsAlert:0, nbRecordsNoAlert:0}' % (indicator_id, session_id) payload = '{"query": "mutation {createIndicatorResult (input: %s) {indicatorResult {indicator {name} session {batch {batchOwner {name}}}}}}"}' % input response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() indicator_name = json['data']['createIndicatorResult']['indicatorResult']['indicator']['name'] batch_owner_name = json['data']['createIndicatorResult']['indicatorResult']['session']['batch']['batchOwner']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(indicator_name, test_case_name) self.assertEqual(batch_owner_name, test_case_name)
def test_mutation_create_batch(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({"class": "ModelBatchOwner", "test_case": test_case_name}) # Create batch owner payload = '{"query": "mutation {createBatchOwner (input: {name:\\"%s\\"}) {batchOwner {id}}}"}' % test_case_name response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() batch_owner_id = json['data']['createBatchOwner']['batchOwner']['id'] # Create batch status_id = to_global_id('SchemaStatus', 1) # Running input = '{batchOwnerId:\\"%s\\", statusId: \\"%s\\"}' % (batch_owner_id, status_id) payload = '{"query": "mutation {createBatch (input: %s) {batch {batchOwner {name} status {name}}}}"}' % (input) response = requests.post(self.base_url, headers=self.headers, data=payload) json = response.json() batch_owner_name = json['data']['createBatch']['batch']['batchOwner']['name'] status_name = json['data']['createBatch']['batch']['status']['name'] self.assertEqual(response.status_code, 200) self.assertEqual(batch_owner_name, test_case_name) self.assertEqual(status_name, 'Running')
def test_get_data_frame(self): test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({ 'class': 'ModelDataSource', 'test_case': test_case_name }) # Create data source current_directory = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) parent_directory = os.path.dirname(current_directory) Operation('ModelDataSource').create( name=test_case_name, dataSourceTypeId=6, # SQLite connectionString=parent_directory + '/database/data_quality.db', login='', password='') # Get data frame request = "select 'status', count(*) as nb_records from status" data_frame = MethodDataSource(test_case_name).get_data_frame(request) for row in data_frame.index: self.assertGreaterEqual(data_frame.loc[row, 'nb_records'], 0)
def test_execute_freshness(self): """Test execute freshness indicator.""" test_case_name = test_utils.get_test_case_name(self.test_case_list) self.test_case_list.append({ 'class': 'ModelIndicator', 'test_case': test_case_name }) self.test_case_list.append({ 'class': 'ModelDataSource', 'test_case': test_case_name }) self.test_case_list.append({ 'class': 'ModelBatchOwner', 'test_case': test_case_name }) # Create batch owner batch_owner = Operation('ModelBatchOwner').create(name=test_case_name) # Create data source current_directory = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) parent_directory = os.path.dirname(current_directory) data_source = Operation('ModelDataSource').create( name=test_case_name, dataSourceTypeId=6, # SQLite connectionString=parent_directory + '/database/data_quality.db', login='', password='') # Create indicator indicator_record = Operation('ModelIndicator').create( name=test_case_name, description=test_case_name, indicatorTypeId=2, # Freshness batchOwnerId=batch_owner.id, executionOrder=0, active=1) # Create indicator paramters param = Operation('ModelIndicatorParameter') param.create(parameterTypeId=1, value=">=", indicatorId=indicator_record.id) # Alert operator param.create(parameterTypeId=2, value="0", indicatorId=indicator_record.id) # Alert threshold param.create(parameterTypeId=3, value="['last_updated_date']", indicatorId=indicator_record.id) # Measures param.create(parameterTypeId=4, value="['table_name']", indicatorId=indicator_record.id) # Dimensions param.create(parameterTypeId=5, value=data_source.name, indicatorId=indicator_record.id) # Target param.create(parameterTypeId=6, value="select 'status', max(updated_date) from status", indicatorId=indicator_record.id) # Target request param.create(parameterTypeId=9, value="['*****@*****.**']", indicatorId=indicator_record.id) # Distribution list # Start batch, execute indicator and stop batch batch_record = MethodBatch(batch_owner.id).start() MethodIndicator(indicator_record.id).execute(batch_record.id) MethodBatch(batch_owner.id).stop(batch_record.id) session = Operation('ModelSession').read( indicatorId=indicator_record.id, batchId=batch_record.id) self.assertEqual(session[0].statusId, 2)
def setUpClass(self): """Set up function called when class is consructed.""" self.base_url = 'http://{}:5000/dataquality/api/graphql'.format( socket.gethostname()) self.headers = {'content-type': 'application/json'} self.test_case_name = test_utils.get_test_case_name([]) self.test_case_list = [self.test_case_name] # Create test data set for subsequent query test cases # Create data source data_source_data = {} data_source_data['name'] = self.test_case_name data_source_data['dataSourceTypeId'] = 1 # Hive data_source_data['connectionString'] = self.test_case_name data_source_data['login'] = self.test_case_name data_source_data['password'] = self.test_case_name self.data_source = Operation('ModelDataSource').create( **data_source_data) # Create batch owner batch_owner_data = {} batch_owner_data['name'] = self.test_case_name self.batch_owner = Operation('ModelBatchOwner').create( **batch_owner_data) # Create indicator indicator_data = {} indicator_data['name'] = self.test_case_name indicator_data['description'] = self.test_case_name indicator_data['indicatorTypeId'] = 1 # Completeness indicator_data['batchOwnerId'] = self.batch_owner.id indicator_data['executionOrder'] = 1 indicator_data['active'] = True self.indicator = Operation('ModelIndicator').create(**indicator_data) # Create indicator parameter parameter_data = {} parameter_data['indicatorId'] = self.indicator.id parameter_data['parameterTypeId'] = 6 # Target request parameter_data['value'] = self.test_case_name self.parameter = Operation('ModelIndicatorParameter').create( **parameter_data) # Create batch batch_data = {} batch_data['batchOwnerId'] = self.batch_owner.id batch_data['statusId'] = 1 # Running self.batch = Operation('ModelBatch').create(**batch_data) # Create session session_data = {} session_data['indicatorId'] = self.indicator.id session_data['batchId'] = self.batch.id session_data['statusId'] = 1 # Running self.session = Operation('ModelSession').create(**session_data) # Create event event_data = {} event_data['sessionId'] = self.session.id event_data['eventTypeId'] = 1 # Start self.event = Operation('ModelEvent').create(**event_data) # Create indicator result result_data = {} result_data['indicatorId'] = self.indicator.id result_data['sessionId'] = self.session.id result_data['alertOperator'] = '=' result_data['alertThreshold'] = 0 result_data['nbRecords'] = 0 result_data['nbRecordsAlert'] = 0 result_data['nbRecordsNoAlert'] = 0 self.result = Operation('ModelIndicatorResult').create(**result_data)