def test_execute_query(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) with self.app.test_request_context(): common_actions.add_auth_to_session(flask.session, self.project.users[0]) query_string2 = 'labels.apple > 3' query_creator = QueryCreator(session=self.session, project=self.project, member=self.member) # Correct case diffgram_query_obj = query_creator.create_query( query_string=query_string2) executor = SqlAlchemyQueryExecutor( session=self.session, diffgram_query=diffgram_query_obj) sql_alchemy_query, execution_log = executor.execute_query() self.assertEqual(type(sql_alchemy_query), Query) self.assertEqual(len(executor.log['error'].keys()), 0)
def test_task_template_launch_core(self): # Create mock tasks label = data_mocking.create_label({ 'name': 'mylabel', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) file = data_mocking.create_file({'project_id': self.project.id}, self.session) job = data_mocking.create_job( { 'name': 'my-test-job-{}'.format(1), 'project': self.project, 'type': "Normal", }, self.session) result = task_template_launch_handler.task_template_launch_core( self.session, job) self.assertEqual(result, job) result = task_template_launch_handler.task_template_launch_core( self.session, None) self.session.commit() self.assertEqual(result, False) job = Job.get_by_id(self.session, job_id=job.id) self.assertEqual(job.status, 'active')
def test_compare_expr(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) with self.app.test_request_context(): common_actions.add_auth_to_session(flask.session, self.project.users[0]) query_string2 = 'labels.apple > 3' query_creator = QueryCreator(session=self.session, project=self.project, member=self.member) # Correct case diffgram_query_obj = query_creator.create_query( query_string=query_string2) executor = SqlAlchemyQueryExecutor( session=self.session, diffgram_query=diffgram_query_obj) sql_alchemy_query, execution_log = executor.execute_query() result = executor.compare_expr(diffgram_query_obj.tree.children[0]. children[0].children[0].children[0]) self.assertEqual(len(execution_log['error'].keys()), 0) self.assertIsNotNone(result) self.assertIsNotNone(result.query_condition) self.assertTrue(executor.valid)
def test__parse_value(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) with self.app.test_request_context(): common_actions.add_auth_to_session(flask.session, self.project.users[0]) query_string = 'labels.x > 5' # dummy query query_creator = QueryCreator(session=self.session, project=self.project, member=self.member) diffgram_query_obj = query_creator.create_query( query_string=query_string) executor = SqlAlchemyQueryExecutor( session=self.session, diffgram_query=diffgram_query_obj) token = Token(value='labels.apple', type_='dummy') value = executor._SqlAlchemyQueryExecutor__parse_value(token) self.assertEqual(type(value), ScalarSelect) token = Token(value='file.something', type_='dummy') value = executor._SqlAlchemyQueryExecutor__parse_value(token) self.assertEqual(type(value), BinaryExpression)
def test_create_query(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) query_str = 'labels.apple > 0' with self.app.test_request_context(): common_actions.add_auth_to_session(flask.session, self.project.users[0]) creator = QueryCreator(session=self.session, project=self.project, member=self.member) result = creator.create_query(query_string=query_str) self.assertTrue(type(result) == DiffgramQuery) self.assertEqual(result.tree.data, 'start') self.assertEqual(result.tree.children[0].data, 'expr') self.assertEqual(result.tree.children[0].children[0].data, 'term') self.assertEqual( result.tree.children[0].children[0].children[0].data, 'factor') self.assertEqual( result.tree.children[0].children[0].children[0].children[0]. data, 'compare_expr')
def test_get_suggestions(self): query_str = '' with self.app.test_request_context(): common_actions.add_auth_to_session(flask.session, self.project.users[0]) creator = QueryCreator(self.session, self.project, self.member) result, type = creator.get_suggestions(query_string=query_str) self.assertTrue('labels' in result) self.assertTrue('file' in result) self.assertEqual(type, 'entities') label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) label2 = data_mocking.create_label({ 'name': 'car', }, self.session) label_file2 = data_mocking.create_label_file( { 'label': label2, 'project_id': self.project.id }, self.session) label3 = data_mocking.create_label({ 'name': 'person', }, self.session) label_file3 = data_mocking.create_label_file( { 'label': label3, 'project_id': self.project.id }, self.session) query_str = 'labels.' result, type = creator.get_suggestions(query_string=query_str) self.assertEqual(type, 'labels') self.assertTrue('apple' in result) self.assertTrue('car' in result) self.assertTrue('person' in result) query_str = 'file.' result, type = creator.get_suggestions(query_string=query_str) self.assertEqual(type, 'labels')
def test__validate_expression(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) with self.app.test_request_context(): common_actions.add_auth_to_session(flask.session, self.project.users[0]) query_string = 'labels.x > 5' # dummy query query_creator = QueryCreator(session=self.session, project=self.project, member=self.member) diffgram_query_obj = query_creator.create_query( query_string=query_string) executor = SqlAlchemyQueryExecutor( session=self.session, diffgram_query=diffgram_query_obj) token = Token(value='something', type_='dummy') token2 = Token(value='5', type_='dummy') operator = Token(value='>', type_='dummy') value = executor._SqlAlchemyQueryExecutor__validate_expression( token, token2, operator) self.assertFalse(value, False) self.assertIsNotNone(executor.log['error'].get('compare_expr')) token = Token(value='labels.apple', type_='dummy') token2 = Token(value='5', type_='dummy') operator = Token(value='>', type_='dummy') executor.log['error'] = {} value = executor._SqlAlchemyQueryExecutor__validate_expression( token, token2, operator) self.assertTrue(value) self.assertEqual(len(executor.log['error'].keys()), 0) token = Token(value='file.sensor', type_='dummy') token2 = Token(value='sensorA', type_='dummy') operator = Token(value='>', type_='dummy') executor.log['error'] = {} value = executor._SqlAlchemyQueryExecutor__validate_expression( token, token2, operator) self.assertFalse(value) self.assertEqual(len(executor.log['error'].keys()), 1)
def test_batch_new_api(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) request_data = { 'pre_labeled_data': { 'myfile.jpg': [{ 'type': 'box', 'label': 'apple', 'x_min': 1, 'x_max': 18, 'y_min': 1, 'y_max': 17, }] } } auth_api = common_actions.create_project_auth(project=self.project, session=self.session) credentials = b64encode( "{}:{}".format(auth_api.client_id, auth_api.client_secret).encode()).decode('utf-8') endpoint = "/api/v1/project/{}/input-batch/new".format( self.project.project_string_id) response = self.client.post(endpoint, data=json.dumps(request_data), headers={ 'directory_id': str(self.project.directory_default_id), 'Authorization': 'Basic {}'.format(credentials) }) data = response.json self.assertEqual(response.status_code, 200) self.assertIsNotNone(data.get('input_batch'))
def test_execute_after_launch_strategy(self): file = data_mocking.create_file( { 'project_id': self.project.id, 'type': 'text' }, self.session) label = data_mocking.create_label({ 'name': 'mylabel', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) attach_dir1 = data_mocking.create_directory( { 'project': self.project, 'user': self.project_data['users'][0], 'files': [file] }, self.session) connection = data_mocking.create_connection( { 'name': 'test', 'integration_name': 'datasaur', 'project_id': self.project.id }, self.session) labeldict = { "label_file_list_serialized": [{ "id": label_file.id, "hash": "083e9ebc48d64e9a8874c6b95f490b56b8c4c5b0f4dacd90bd3534085e87d9fa", "type": "label", "state": "added", "created_time": "2020-07-15T18:48:34.477333", "time_last_updated": "2020-07-15T18:48:34.705290", "ann_is_complete": None, "original_filename": None, "video_id": None, "video_parent_file_id": None, "count_instances_changed": None, "attribute_group_list": [{ "id": 2, "kind": "multiple_select", "is_root": True, "name": "carwheeltag", "prompt": "How is this car wheel", "show_prompt": True, "time_updated": "2020-08-05 19:37:07.703576", "attribute_template_list": [{ "id": 4, "name": "Is rounded", "value_type": None, "archived": False, "group_id": 2, "display_order": None }, { "id": 5, "name": "is squared", "value_type": None, "archived": False, "group_id": 2, "display_order": None }, { "id": 6, "name": "is beautiful", "value_type": None, "archived": False, "group_id": 2, "display_order": None }, { "id": 7, "name": "is crazy", "value_type": None, "archived": False, "group_id": 2, "display_order": None }] }, { "id": 3, "kind": "select", "is_root": True, "name": "selectwheel", "prompt": "Please selectt something special about this wheels", "show_prompt": True, "time_updated": "2020-08-12 16:29:54.817801", "attribute_template_list": [{ "id": 10, "name": "Silver Wheel", "value_type": None, "archived": False, "group_id": 3, "display_order": None }, { "id": 9, "name": "+Gold wheel", "value_type": None, "archived": False, "group_id": 3, "display_order": None }] }, { "id": 4, "kind": "text", "is_root": True, "name": "freewheel", "prompt": "What are your thought on this wheel?", "show_prompt": True, "time_updated": "2020-08-05 20:50:59.195249", "attribute_template_list": [] }, { "id": 5, "kind": "radio", "is_root": True, "name": "clean", "prompt": "Is this wheel clean?", "show_prompt": True, "time_updated": "2020-08-05 20:53:46.314143", "attribute_template_list": [{ "id": 11, "name": "Wheel is dirty", "value_type": None, "archived": False, "group_id": 5, "display_order": None }, { "id": 12, "name": "Wheek is clean", "value_type": None, "archived": False, "group_id": 5, "display_order": None }] }, { "id": 6, "kind": "text", "is_root": True, "name": "TEST", "prompt": "TEST28", "show_prompt": True, "time_updated": "2020-08-12 16:30:03.770141", "attribute_template_list": [] }], "colour": { "hex": "#194d33", "hsl": { "h": 150, "s": 0.5, "l": 0.2, "a": 1 }, "hsv": { "h": 150, "s": 0.66, "v": 0.3, "a": 1 }, "rgba": { "r": 25, "g": 77, "b": 51, "a": 1 }, "a": 1 }, "label": { "id": 5, "name": "Car wheel", "default_sequences_to_single_frame": False } }], "label_file_colour_map": {} } job = data_mocking.create_job( { 'name': 'my-test-job-{}'.format(1), 'project': self.project, 'status': 'active', 'type': "Normal", 'label_dict': labeldict, 'attached_directories': [attach_dir1], 'interface_connection_id': connection.id }, self.session) strategy = DatasaurTaskTemplateAfterLaunchStrategy( task_template=job, session=self.session, log=regular_log.default()) with patch.object(DatasaurTaskTemplateAfterLaunchStrategy, 'create_datasaur_labelset', return_value={ 'result': { 'createLabelSet': { 'id': 'mytestid' } } }): with patch.object(DatasaurTaskTemplateAfterLaunchStrategy, 'create_datasaur_project', return_value={'result': { 'id': 'datasaur_test' }}): with patch.object(DatasaurTaskTemplateAfterLaunchStrategy, 'get_project_files_list', return_value={ 'result': { 'id': 'datasaur_test', 'documents': [{ 'id': str(file.id), 'name': str(file.id) }] } }): strategy.execute_after_launch_strategy() commit_with_rollback(self.session) tasks_count = self.session.query(Task).filter( Task.job_id == job.id).count() tasks = self.session.query(Task).filter( Task.job_id == job.id).all() self.assertEqual(tasks_count, 1) external_map = ExternalMap.get( session=self.session, job_id=job.id, external_id='mytestid', connection_id=connection.id, diffgram_class_string='', type='{}_label_set'.format( connection.integration_name), ) self.assertNotEqual(external_map, None) project_map = ExternalMap.get( session=self.session, job_id=job.id, external_id='datasaur_test', connection_id=connection.id, diffgram_class_string='task_template', type='{}_project'.format(connection.integration_name), ) self.assertNotEqual(project_map, None) files_maps = ExternalMap.get( session=self.session, job_id=job.id, external_id=str(file.id), file_id=file.id, connection_id=connection.id, diffgram_class_string='file', type='{}_file'.format(connection.integration_name), ) self.assertNotEqual(files_maps, None)
def test_api_create_sequence_preview(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) video_file = data_mocking.create_file( { 'project_id': self.project.id, 'type': 'video' }, self.session) sequence = data_mocking.create_sequence( { 'label_file_id': label_file.id, 'video_file_id': video_file.id, 'cache_expiry': time.time() + 500000, 'number': 1, }, self.session) video_file_bad = data_mocking.create_file( { 'project_id': self.project2.id, 'type': 'video' }, self.session) preview_url = 'https://picsum.photos/200/300' instance = data_mocking.create_instance( { 'project_id': self.project.id, 'type': 'box', 'x_min': 0, 'x_max': 0, 'y_min': 0, 'y_max': 0, 'file_id': video_file.id, 'soft_delete': False, 'sequence_id': sequence.id, 'preview_image_url': preview_url, 'preview_image_url_expiry': 900000000, }, self.session) sequence.instance_preview_cache = { 'id': instance.id, 'file_id': sequence.video_file.id, 'preview_image_url': preview_url, } self.session.commit() endpoint = "/api/project/{}/sequence/{}/create-preview".format( self.project.project_string_id, sequence.id, ) auth_api = common_actions.create_project_auth(project=self.project, session=self.session) credentials = b64encode( "{}:{}".format(auth_api.client_id, auth_api.client_secret).encode()).decode('utf-8') response = self.client.post(endpoint, data=json.dumps({}), headers={ 'directory_id': str(self.project.directory_default_id), 'Authorization': 'Basic {}'.format(credentials) }) data = response.json self.assertTrue('result' in data) self.assertTrue('log' in data) self.assertFalse(len(data['log']['error'].keys()), 0) self.assertEqual(data['result']['instance_preview']['id'], instance.id) self.assertEqual(data['result']['instance_preview']['file_id'], video_file.id) self.assertEqual( data['result']['instance_preview']['preview_image_url'], preview_url) # Error case sequence2 = data_mocking.create_sequence( { 'label_file_id': label_file.id, 'video_file_id': video_file_bad.id, 'cache_expiry': time.time() + 500000, 'number': 1, }, self.session) result, log = sequence_preview_create.create_sequence_preview_core( session=self.session, log=regular_log.default(), project=self.project, sequence_id=sequence2.id) self.assertEqual(len(log['error'].keys()), 1) self.assertTrue('project_id' in log['error'])
def test_create_sequence_preview_core(self): label = data_mocking.create_label({ 'name': 'apple', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) video_file = data_mocking.create_file( { 'project_id': self.project.id, 'type': 'video' }, self.session) video_file_bad = data_mocking.create_file( { 'project_id': self.project2.id, 'type': 'video' }, self.session) sequence = data_mocking.create_sequence( { 'label_file_id': label_file.id, 'video_file_id': video_file.id, 'cache_expiry': time.time() + 500000, 'number': 1, }, self.session) sequence2 = data_mocking.create_sequence( { 'label_file_id': label_file.id, 'video_file_id': video_file_bad.id, 'cache_expiry': time.time() + 500000, 'number': 1, }, self.session) preview_url = 'https://picsum.photos/200/300' instance = data_mocking.create_instance( { 'project_id': self.project.id, 'type': 'box', 'x_min': 0, 'x_max': 0, 'y_min': 0, 'y_max': 0, 'file_id': video_file.id, 'soft_delete': False, 'sequence_id': sequence.id, 'preview_image_url': preview_url, 'preview_image_url_expiry': 900000000, }, self.session) sequence.instance_preview_cache = { 'id': instance.id, 'file_id': sequence.video_file.id, 'preview_image_url': preview_url, } self.session.commit() result, log = sequence_preview_create.create_sequence_preview_core( session=self.session, log=regular_log.default(), project=self.project, sequence_id=sequence.id) self.assertFalse(len(log['error'].keys()), 0) self.assertEqual(result['instance_preview']['id'], instance.id) self.assertEqual(result['instance_preview']['file_id'], video_file.id) self.assertEqual(result['instance_preview']['preview_image_url'], preview_url) # Error case result, log = sequence_preview_create.create_sequence_preview_core( session=self.session, log=regular_log.default(), project=self.project, sequence_id=sequence2.id) self.assertEqual(len(log['error'].keys()), 1) self.assertTrue('project_id' in log['error'])