def test_execute_query(self):
label = data_mocking.create_label({
'name': 'apple',
}, self.session)
label_file = data_mocking.create_label_file(
{
'label': label,
'project_id': self.project.id
}, self.session)
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
query_string2 = 'labels.apple > 3'
query_creator = QueryCreator(session=self.session,
project=self.project,
member=self.member)
# Correct case
diffgram_query_obj = query_creator.create_query(
query_string=query_string2)
executor = SqlAlchemyQueryExecutor(
session=self.session, diffgram_query=diffgram_query_obj)
sql_alchemy_query, execution_log = executor.execute_query()
self.assertEqual(type(sql_alchemy_query), Query)
self.assertEqual(len(executor.log['error'].keys()), 0)
def test_job_view_core(self):
num_jobs = 3
metadata_proposed = {
'builder_or_trainer': {
'mode': 'builder'
},
'limit': num_jobs,
'my_jobs_only': False,
'project_string_id': self.project.project_string_id
}
all_jobs = []
for i in range(0, num_jobs):
job = data_mocking.create_job(
{
'name': 'my-test-job-{}'.format(i),
'project': self.project
}, self.session)
all_jobs.append(job)
self.session.commit()
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
result = job_view_core(self.session,
metadata_proposed,
output_mode="serialize",
user=self.project.users[0])
logger.info(result)
self.assertEqual(len(result), num_jobs - 1)
def test_compare_expr(self):
label = data_mocking.create_label({
'name': 'apple',
}, self.session)
label_file = data_mocking.create_label_file(
{
'label': label,
'project_id': self.project.id
}, self.session)
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
query_string2 = 'labels.apple > 3'
query_creator = QueryCreator(session=self.session,
project=self.project,
member=self.member)
# Correct case
diffgram_query_obj = query_creator.create_query(
query_string=query_string2)
executor = SqlAlchemyQueryExecutor(
session=self.session, diffgram_query=diffgram_query_obj)
sql_alchemy_query, execution_log = executor.execute_query()
result = executor.compare_expr(diffgram_query_obj.tree.children[0].
children[0].children[0].children[0])
self.assertEqual(len(execution_log['error'].keys()), 0)
self.assertIsNotNone(result)
self.assertIsNotNone(result.query_condition)
self.assertTrue(executor.valid)
def test__parse_value(self):
label = data_mocking.create_label({
'name': 'apple',
}, self.session)
label_file = data_mocking.create_label_file(
{
'label': label,
'project_id': self.project.id
}, self.session)
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
query_string = 'labels.x > 5' # dummy query
query_creator = QueryCreator(session=self.session,
project=self.project,
member=self.member)
diffgram_query_obj = query_creator.create_query(
query_string=query_string)
executor = SqlAlchemyQueryExecutor(
session=self.session, diffgram_query=diffgram_query_obj)
token = Token(value='labels.apple', type_='dummy')
value = executor._SqlAlchemyQueryExecutor__parse_value(token)
self.assertEqual(type(value), ScalarSelect)
token = Token(value='file.something', type_='dummy')
value = executor._SqlAlchemyQueryExecutor__parse_value(token)
self.assertEqual(type(value), BinaryExpression)
def test_create_query(self):
label = data_mocking.create_label({
'name': 'apple',
}, self.session)
label_file = data_mocking.create_label_file(
{
'label': label,
'project_id': self.project.id
}, self.session)
query_str = 'labels.apple > 0'
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
creator = QueryCreator(session=self.session,
project=self.project,
member=self.member)
result = creator.create_query(query_string=query_str)
self.assertTrue(type(result) == DiffgramQuery)
self.assertEqual(result.tree.data, 'start')
self.assertEqual(result.tree.children[0].data, 'expr')
self.assertEqual(result.tree.children[0].children[0].data, 'term')
self.assertEqual(
result.tree.children[0].children[0].children[0].data, 'factor')
self.assertEqual(
result.tree.children[0].children[0].children[0].children[0].
data, 'compare_expr')
def test_filter_by_project(self):
other_project_data = data_mocking.create_project_with_context(
{
'users': [{
'username': '******',
'email': '*****@*****.**',
'password': '******',
}]
}, self.session)
other_project = other_project_data['project']
other_job = data_mocking.create_job(
{
'name': 'my-testother-job-{}'.format(1),
'project': other_project
}, self.session)
query = self.session.query(Job)
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
result = filter_by_project(
session=self.session,
project_string_id=self.project.project_string_id,
query=query)
jobs = result.all()
job_ids = [x.id for x in jobs]
self.assertTrue(other_job.id not in job_ids)
def test_task_next_issue(self):
# Create mock tasks
num_tasks = 5
job = data_mocking.create_job(
{
'name': 'my-test-job-{}'.format(1),
'project': self.project
}, self.session)
all_tasks = []
for i in range(0, num_tasks):
task = data_mocking.create_task(
{
'name': 'task{}'.format(i),
'job': job
}, self.session)
all_tasks.append(task)
issue1 = data_mocking.create_discussion(
{
'title': 'test',
'description': 'test',
'project_id': self.project.id
}, self.session)
issue1.attach_element(session=self.session,
element={
'type': 'task',
'id': all_tasks[2].id
})
issue2 = data_mocking.create_discussion(
{
'title': 'test2',
'description': 'test2',
'project_id': self.project.id
}, self.session)
issue2.attach_element(session=self.session,
element={
'type': 'task',
'id': all_tasks[4].id
})
with self.client.session_transaction() as session:
endpoint = "/api/v1/task/{}/next-task-with-issues".format(
all_tasks[0].id)
credentials = b64encode("{}:{}".format(
self.auth_api.client_id,
self.auth_api.client_secret).encode()).decode('utf-8')
session['Authorization'] = credentials
common_actions.add_auth_to_session(session, self.project.users[0])
response = self.client.post(endpoint,
data=json.dumps({}),
headers={
'directory_id':
str(self.project.directory_default_id),
'Authorization':
'Basic {}'.format(credentials)
})
data = response.json
self.assertEqual(response.status_code, 200)
self.assertEqual(data['task_id'], all_tasks[2].id)
def test_get_video_frame_from_task(self):
# Create mock task
job = data_mocking.create_job({
'name': 'my-test-job-{}'.format(1),
'project': self.project
}, self.session)
video_file = data_mocking.create_file({
'project_id': self.project.id,
'type': 'video'
}, self.session)
frames_list = []
# Mock Frames
for i in range(0, 10):
frame = data_mocking.create_file({
'project_id': self.project.id,
'video_parent_file_id': video_file.id,
'frame_number': i,
'type': 'frame'
}, self.session)
frames_list.append(frame)
task = data_mocking.create_task({
'name': 'tasktest',
'job': job,
'file': video_file
}, self.session)
request_data = {
'frame_list': [1, 2, 3],
'project_string_id': self.project.project_string_id,
'mode_data': 'list'
}
endpoint = "/api/v1/task/{}/video/single/{}/frame-list/".format(task.id, video_file.id)
with self.client.session_transaction() as session:
auth_api = common_actions.create_project_auth(project = self.project, session = self.session)
credentials = b64encode("{}:{}".format(auth_api.client_id, auth_api.client_secret).encode()).decode(
'utf-8')
session['Authorization'] = credentials
common_actions.add_auth_to_session(session, self.project.users[0])
response = self.client.post(
endpoint,
data = json.dumps(request_data),
headers = {
'directory_id': str(job.project.directory_default_id),
'Authorization': 'Basic {}'.format(credentials)
}
)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.json['url_list']), 3)
i = 1
for elm in response.json['url_list']:
self.assertEqual(elm['frame_number'], i)
i += 1
def test_sync_events_list_api(self):
# Create mock tasks
num_events = 5
job = data_mocking.create_job(
{
'name': 'my-test-job-{}'.format(1),
'project': self.project
}, self.session)
all_sync_events = []
for i in range(0, num_events):
sync_event = data_mocking.create_sync_event(
{
'description': 'syncevent{}'.format(i),
'job_id': job.id,
'project': self.project
}, self.session)
all_sync_events.append(sync_event)
request_payload = {
'metadata': {
'job_id': job.id,
'mode_data': 'list',
'project_string_id': self.project.project_string_id
}
}
endpoint = "/api/v1/sync-events/list"
with self.client.session_transaction() as session:
auth_api = common_actions.create_project_auth(project=self.project,
session=self.session)
credentials = b64encode("{}:{}".format(
auth_api.client_id,
auth_api.client_secret).encode()).decode('utf-8')
session['Authorization'] = credentials
common_actions.add_auth_to_session(session, self.project.users[0])
response = self.client.post(endpoint,
data=json.dumps(request_payload),
headers={
'directory_id':
str(job.project.directory_default_id),
'Authorization':
'Basic {}'.format(credentials)
})
print(response.data)
self.assertEqual(response.status_code, 200)
new_session = sessionMaker.session_factory()
self.assertEqual(len(response.json['sync_events_list']), num_events)
def test_get_suggestions(self):
query_str = ''
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
creator = QueryCreator(self.session, self.project, self.member)
result, type = creator.get_suggestions(query_string=query_str)
self.assertTrue('labels' in result)
self.assertTrue('file' in result)
self.assertEqual(type, 'entities')
label = data_mocking.create_label({
'name': 'apple',
}, self.session)
label_file = data_mocking.create_label_file(
{
'label': label,
'project_id': self.project.id
}, self.session)
label2 = data_mocking.create_label({
'name': 'car',
}, self.session)
label_file2 = data_mocking.create_label_file(
{
'label': label2,
'project_id': self.project.id
}, self.session)
label3 = data_mocking.create_label({
'name': 'person',
}, self.session)
label_file3 = data_mocking.create_label_file(
{
'label': label3,
'project_id': self.project.id
}, self.session)
query_str = 'labels.'
result, type = creator.get_suggestions(query_string=query_str)
self.assertEqual(type, 'labels')
self.assertTrue('apple' in result)
self.assertTrue('car' in result)
self.assertTrue('person' in result)
query_str = 'file.'
result, type = creator.get_suggestions(query_string=query_str)
self.assertEqual(type, 'labels')
def test__validate_expression(self):
label = data_mocking.create_label({
'name': 'apple',
}, self.session)
label_file = data_mocking.create_label_file(
{
'label': label,
'project_id': self.project.id
}, self.session)
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
query_string = 'labels.x > 5' # dummy query
query_creator = QueryCreator(session=self.session,
project=self.project,
member=self.member)
diffgram_query_obj = query_creator.create_query(
query_string=query_string)
executor = SqlAlchemyQueryExecutor(
session=self.session, diffgram_query=diffgram_query_obj)
token = Token(value='something', type_='dummy')
token2 = Token(value='5', type_='dummy')
operator = Token(value='>', type_='dummy')
value = executor._SqlAlchemyQueryExecutor__validate_expression(
token, token2, operator)
self.assertFalse(value, False)
self.assertIsNotNone(executor.log['error'].get('compare_expr'))
token = Token(value='labels.apple', type_='dummy')
token2 = Token(value='5', type_='dummy')
operator = Token(value='>', type_='dummy')
executor.log['error'] = {}
value = executor._SqlAlchemyQueryExecutor__validate_expression(
token, token2, operator)
self.assertTrue(value)
self.assertEqual(len(executor.log['error'].keys()), 0)
token = Token(value='file.sensor', type_='dummy')
token2 = Token(value='sensorA', type_='dummy')
operator = Token(value='>', type_='dummy')
executor.log['error'] = {}
value = executor._SqlAlchemyQueryExecutor__validate_expression(
token, token2, operator)
self.assertFalse(value)
self.assertEqual(len(executor.log['error'].keys()), 1)
def test_job_list_api(self):
# Create mock job.
num_jobs = 5
all_jobs = []
for i in range(0, num_jobs):
job = data_mocking.create_job(
{
'name': 'my-test-job-{}'.format(i),
'project': self.project
}, self.session)
all_jobs.append(job)
request_data = {
'metadata': {
'builder_or_trainer': {
'mode': 'builder'
},
'limit': 5,
'project_string_id': self.project.project_string_id
}
}
endpoint = "/api/v1/job/list"
with self.client.session_transaction() as session:
auth_api = common_actions.create_project_auth(project=self.project,
session=self.session)
credentials = b64encode("{}:{}".format(
auth_api.client_id,
auth_api.client_secret).encode()).decode('utf-8')
session['Authorization'] = credentials
common_actions.add_auth_to_session(session, self.project.users[0])
response = self.client.post(endpoint,
data=json.dumps(request_data),
headers={
'directory_id':
str(job.project.directory_default_id),
'Authorization':
'Basic {}'.format(credentials)
})
self.assertEqual(response.status_code, 200)
new_session = sessionMaker.session_factory()
self.assertEqual(len(response.json['Job_list']), num_jobs)
def test_job_pin_api(self):
# Create mock job.
job = data_mocking.create_job(
{
'name': 'my-test-job',
'project': self.project
}, self.session)
request_data = {}
endpoint = "/api/v1/job/{}/pin".format(job.id)
with self.client.session_transaction() as session:
auth_api = common_actions.create_project_auth(project=self.project,
session=self.session)
credentials = b64encode("{}:{}".format(
auth_api.client_id,
auth_api.client_secret).encode()).decode('utf-8')
session['Authorization'] = credentials
common_actions.add_auth_to_session(session, self.project.users[0])
response = self.client.post(endpoint,
data=json.dumps(request_data),
headers={
'directory_id':
str(job.project.directory_default_id),
'Authorization':
'Basic {}'.format(credentials)
})
data = response.json
self.assertEqual(response.status_code, 200)
self.assertEqual(data['job']['is_pinned'], True)
response = self.client.post(endpoint,
data=json.dumps(request_data),
headers={
'directory_id':
str(job.project.directory_default_id),
'Authorization':
'Basic {}'.format(credentials)
})
data = response.json
self.assertEqual(response.status_code, 200)
self.assertEqual(data['job']['is_pinned'], False)
def test_sync_events_list_core(self):
# Create mock tasks
num_tasks = 5
job = data_mocking.create_job(
{
'name': 'my-test-job-{}'.format(2),
'project': self.project
}, self.session)
all_tasks = []
for i in range(0, num_tasks):
sync_event = data_mocking.create_sync_event(
{
'description': 'syncevent{}'.format(i),
'job_id': job.id,
'project': self.project
}, self.session)
all_tasks.append(sync_event)
self.session.commit()
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
result = sync_events_list_core(
self.session,
date_from=None,
date_to=None,
status=None,
job_id=job.id,
dataset_source_id=None,
dataset_destination_id=None,
created_task_id=None,
completed_task_id=None,
event_effect_type=None,
event_trigger_type=None,
incoming_directory_id=None,
project_string_id=self.project.project_string_id,
limit=30,
output_mode='serialize')
logger.info(result)
self.assertEqual(len(result), num_tasks)
def test_get_compare_op(self):
query_string = 'labels.x > 5' # dummy query
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
query_creator = QueryCreator(session=self.session,
project=self.project,
member=self.member)
diffgram_query_obj = query_creator.create_query(
query_string=query_string)
executor = SqlAlchemyQueryExecutor(
session=self.session, diffgram_query=diffgram_query_obj)
token = Token(value='>=', type_='dummy')
op = executor.get_compare_op(token)
self.assertEqual(op, operator.ge)
token = Token(value='=', type_='dummy')
op = executor.get_compare_op(token)
self.assertEqual(op, operator.eq)
token = Token(value='>=', type_='dummy')
op = executor.get_compare_op(token)
self.assertEqual(op, operator.ge)
token = Token(value='<', type_='dummy')
op = executor.get_compare_op(token)
self.assertEqual(op, operator.lt)
token = Token(value='<=', type_='dummy')
op = executor.get_compare_op(token)
self.assertEqual(op, operator.le)
token = Token(value='!=', type_='dummy')
op = executor.get_compare_op(token)
self.assertEqual(op, operator.ne)
def test__determine_entity_type(self):
token = Token(value='labels.apple', type_='dummy')
token2 = Token(value='files.metadata', type_='dummy')
token3 = Token(value='issues.count', type_='dummy')
query_string = 'labels.x > 5' # dummy query
with self.app.test_request_context():
common_actions.add_auth_to_session(flask.session,
self.project.users[0])
query_creator = QueryCreator(session=self.session,
project=self.project,
member=self.member)
diffgram_query_obj = query_creator.create_query(
query_string=query_string)
executor = SqlAlchemyQueryExecutor(
session=self.session, diffgram_query=diffgram_query_obj)
e_type = executor._SqlAlchemyQueryExecutor__determine_entity_type(
token)
self.assertEqual(e_type, 'labels')
e_type = executor._SqlAlchemyQueryExecutor__determine_entity_type(
token2)
self.assertEqual(e_type, 'files')
e_type = executor._SqlAlchemyQueryExecutor__determine_entity_type(
token3)
self.assertEqual(e_type, 'issues')
