def test_pull_task(): test_app = TestApp(app) session = create_session() task1 = Task() task1.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546018' task1.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546018' task1.claimed = datetime.utcnow() task1.data = json.dumps({'wait_time': 123}) session.add(task1) task2 = Task() task2.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546019' task2.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546019' task2.claimed = datetime.utcnow() task2.completed = datetime.utcnow() task2.result_data = json.dumps({'result': 'epic success'}) session.add(task2) task3 = Task() task3.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546020' task3.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546020' task3.claimed = datetime.utcnow() task3.failed = datetime.utcnow() task3.error = 'unknown error' session.add(task3) session.commit() test_app.get('/task/de305d54-75b4-431b-adb2-eb6b9e546018') test_app.get('/task/de305d54-75b4-431b-adb2-eb6b9e546019') test_app.get('/task/de305d54-75b4-431b-adb2-eb6b9e546020')
def status(): """ Simple status page which verifies that database connectivity works and tells stats about pending tasks and active agents :return: dict in following format { 'agents': 5, # Number of agents that have been active within last Settings.agent_active_threshold seconds 'tasks_waiting': 10 # Number of tasks that haven't been claimed yet } """ try: session = create_session() except: raise HTTPError(500, 'Something wen\'t wrong with database session') agent_time_threshold = datetime.utcnow() - timedelta(0, Settings.agent_active_threshold) try: num_agents = session.query(Agent).filter(Agent.last_seen > agent_time_threshold).count() tasks_waiting = session.query(Task).filter(Task.claimed == None).count() except Exception as e: raise HTTPError(500, 'Failed to query tasks and agents ' + str(e)) finally: session.close() return {'agents': num_agents, 'tasks_waiting': tasks_waiting}
def dev_get_agents(): try: session = create_session() except: raise HTTPError(500) try: query = session.query(Agent) agents = query.all() except Exception as e: logger.exception(e) session.close() raise HTTPError(500) agents_array = [] for agent in agents: agent_json = { 'agent_id': agent.uuid, 'agent_name': agent.name, 'last_seen': str(agent.last_seen) } if agent.capabilities: agent_json['agent_capabilities'] = {} for capability in agent.capabilities: agent_json['agent_capabilities'][capability.type] = { 'version': capability.version } agents_array.append(agent_json) session.close() return {'agents': agents_array}
def test_push_response(): test_app = TestApp(app) session = create_session() task = Task() task.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546013' task.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546013' task.claimed = datetime.utcnow() session.add(task) session.commit() test_app.post_json('/tasks/response', { 'protocol': 1, 'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', 'task_data': { 'key': 'value', 'another_key': 5 } }) session = create_session() task = session.query(Task).filter(Task.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one() assert task.completed is not None assert task.result_data is not None assert task.failed is None assert task.error is None task.completed = None task.result_data = None session.commit() test_app.post_json('/tasks/response', { 'protocol': 1, 'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', 'task_error': 'Something went terribly wrong' }) session = create_session() task = session.query(Task).filter(Task.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one() assert task.completed is None assert task.result_data is None assert task.failed is not None assert task.error is not None session.close()
def test_poll_task_capability_change(): test_app = TestApp(app) test_app.post_json('/tasks', { 'protocol': 1, 'agent_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', 'agent_name': 'Agent 007', 'agent_location': { 'country': 'FI', 'region': '18' }, 'agent_time': '2012-04-23T18:25:43.511Z', 'agent_capabilities': { 'task-type-1': {'version': 1}, 'task-type-2': {'version': 2} }, 'max_tasks': 5 }) test_app.post_json('/tasks', { 'protocol': 1, 'agent_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', 'agent_name': 'Agent 007', 'agent_location': { 'country': 'FI', 'region': '18' }, 'agent_time': '2012-04-23T18:25:43.511Z', 'agent_capabilities': { 'task-type-1': {'version': 1}, 'task-type-2': {'version': 2}, 'task-type-3': {'version': 3}, 'task-type-4': {'version': 4} }, 'max_tasks': 5 }) session = create_session() agent = session.query(Agent).filter(Agent.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one() try: session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\ filter(AgentCapability.type == 'task-type-1').one() session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\ filter(AgentCapability.type == 'task-type-2').one() session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\ filter(AgentCapability.type == 'task-type-3').one() session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\ filter(AgentCapability.type == 'task-type-4').one() finally: session.close()
def get_task(task_uuid: str): """ Gets information about single task with uuid task_uuid :param task_uuid: uuid of the task :return: dict in following format { 'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', # UUID of the task (str) 'test_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', # UUID of the test (str) 'task_type': 'wait', # type of the task (str) 'task_version': 1, # Version number of the task 'task_data': {}, # Dict containing data passed to the task (if any) 'task_completed': '31-03-2015:12:12:12', # Time when task was completed (if completed) 'task_result': {}, # Dict containing task's results (if completed) 'task_failed': '31-03-2015:12:12:12', # Time when task failed (if failed) 'task_error': 'Something went wrong' # Error that caused task to fail (if failed) } """ try: session = create_session() except: raise HTTPError(500) try: query = session.query(Task) task = query.filter(Task.uuid == str(task_uuid)).one() except NoResultFound: raise HTTPError(404) finally: session.close() task_desc = { 'task_id': task.uuid, 'test_id': task.test_id, 'task_type': task.type, 'task_version': task.version } if task.data is not None: task_desc['task_data'] = json.loads(task.data) if task.failed: task_desc['task_failed'] = str(task.failed) task_desc['task_error'] = str(task.error) elif task.completed: task_desc['task_completed'] = str(task.completed) task_desc['task_result'] = json.loads(task.result_data) return task_desc
def post_task(): data = request.json if data is None: raise HTTPError(400) try: jsonschema.validate(data, post_task_schema) except jsonschema.ValidationError: raise HTTPError(400) session = create_session() task_uuid = str(data['task_id']) task_type = str(data['task_type']) task_test_id = (data['test_id']) task_data = "" task = Task( uuid=task_uuid, type=task_type, version=int(data['task_version']), test_id=task_test_id ) if 'task_data' in data: task_data = json.dumps(data['task_data']) task.data = task_data try: session.add(task) except IntegrityError: session.rollback() raise HTTPError(400) try: session.commit() except (IntegrityError, ProgrammingError): session.rollback() logger.error("Failed to commit database changes for BPMS task POST") raise HTTPError(400) finally: session.close() logger.info("Task posted by BPMS - Task's type: {}, test process id: {}, uuid: {}, parameters: {}" .format(task_type, task_test_id, task_uuid, task_data))
def test_post_task_duplicate(): test_app = TestApp(app) session = create_session() task1 = Task() task1.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546018' task1.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546018' task1.claimed = datetime.utcnow() task1.data = json.dumps({'wait_time': 123}) session.add(task1) session.commit() session.close() assert test_app.post_json('/task', { 'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546018', 'test_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', 'task_type': 'wait', 'task_version': 1 }, expect_errors=True).status_int == 400
def dev_post_tasks(): data = request.json if data is None: raise HTTPError(400) try: jsonschema.validate(data, dev_post_tasks_schema) except jsonschema.ValidationError: raise HTTPError(400) try: session = create_session() except: raise HTTPError(500) task = Task( uuid=str(data['task_id']), type=data['task_type'], version=data['task_version'], test_id=data['test_id']) if 'task_data' in data: task.data = json.dumps(data['task_data']) try: session.add(task) except IntegrityError: session.rollback() session.close() raise HTTPError(400) try: session.commit() except (IntegrityError, ProgrammingError): session.rollback() raise HTTPError(400) finally: session.close()
def dev_get_tasks(): try: session = create_session() except: raise HTTPError(500) try: query = session.query(Task) tasks = query.all() except Exception as e: logger.exception(e) raise HTTPError(500) finally: session.close() tasks_array = [] for task in tasks: task_desc = { 'task_id': task.uuid, 'task_type': task.type, 'task_version': task.version, 'test_id': task.test_id } if task.data is not None: task_desc['task_data'] = json.loads(task.data) if task.failed: task_desc['task_failed'] = str(task.failed) task_desc['task_error'] = str(task.error) elif task.completed: task_desc['task_completed'] = str(task.completed) task_desc['task_result'] = str(task.result_data) tasks_array.append(task_desc) return {'tasks': tasks_array}
def request_tasks(): data = request.json if data is None: raise HTTPError(400) try: jsonschema.validate(data, task_request_schema) except jsonschema.ValidationError: raise HTTPError(400) protocol = int(data['protocol']) agent_uuid = str(data['agent_id']) agent_name = str(data['agent_name']) agent_time = data['agent_time'] agent_capabilities = data['agent_capabilities'] max_tasks = int(data['max_tasks']) agent_location = data['agent_location'] if 'agent_location' in data else None # Only protocol 1 supported for now if protocol != 1: raise HTTPError(400) try: session = create_session() except: logger.error("Failed to create database session for task request") raise HTTPError(500) try: query = session.query(Agent) agent = query.filter(Agent.uuid == agent_uuid).one() session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).delete() except NoResultFound: agent = Agent(uuid=agent_uuid, name=agent_name) session.add(agent) for agent_capability, agent_capability_info in agent_capabilities.items(): capability = AgentCapability(type=agent_capability, version=int(agent_capability_info['version']), agent=agent) session.add(capability) # Find all non-claimed tasks that the agent is able to handle and assign them to the agent query = session.query(AgentCapability, Task).filter(Task.assigned_agent_uuid.is_(None)).\ filter(AgentCapability.agent_uuid == agent.uuid).\ filter(and_(AgentCapability.type == Task.type, AgentCapability.version == Task.version)) tasks = [] # Assign available tasks to the agent and mark them as being in process for _, task in query[0:max_tasks]: task.assigned_agent_uuid = agent.uuid task.claimed = datetime.utcnow() tasks.append({ 'task_id': task.uuid, 'task_type': task.type, 'task_version': task.version, 'task_data': json.loads(task.data) }) agent.last_seen = datetime.utcnow() try: session.commit() except Exception: session.rollback() logger.error("Failed to commit database changes for task request") raise HTTPError(500) finally: session.close() logger.info("Agent requested tasks - Agent's name and uuid: {}, {} - Agent was given following tasks: {}" .format(agent_name, agent_uuid, tasks)) return { "tasks": tasks, "return_time": (datetime.now(tz.tzlocal()) + timedelta(0, Settings.agent_return_time)).isoformat() }
def post_tasks(): data = request.json if data is None: logger.error("No JSON content in task response request!") raise HTTPError(400) try: jsonschema.validate(data, task_response_schema) except jsonschema.ValidationError as e: logger.error("Invalid JSON in task reponse: {0}".format(e)) raise HTTPError(400) protocol = int(data['protocol']) task_id = str(data['task_id']) # Only protocol 1 supported for now if protocol != 1: logger.error("Invalid protocol in task response: {0}".format(protocol)) raise HTTPError(400) try: session = create_session() except: logger.error("Failed to create database session for task result POST") raise HTTPError(500) try: task = session.query(Task).filter(Task.uuid == str(task_id)).one() if task.claimed is None or (task.completed is not None or task.failed is not None): logger.error("Incomplete task posted!") session.close() raise HTTPError(400) result = "" if 'task_data' in data: task.result_data = json.dumps(data['task_data']) task.completed = datetime.utcnow() result = json.dumps(task.result_data) elif 'task_error' in data: task.error = data['task_error'] task.failed = datetime.utcnow() result = task.error session.add(task) except NoResultFound: logger.error("No matching task in for task response!") session.close() raise HTTPError(400) try: session.commit() except Exception: session.rollback() logger.error("Failed to commit database changes for task result POST") raise HTTPError(500) finally: session.close() logger.info("An agent returned task with results - uuid: {}, end results: {}".format(task_id, result))
def request_tasks(): data = request.json if data is None: raise HTTPError(400) try: jsonschema.validate(data, task_request_schema) except jsonschema.ValidationError: raise HTTPError(400) protocol = int(data['protocol']) agent_uuid = str(data['agent_id']) agent_name = str(data['agent_name']) agent_time = data['agent_time'] agent_capabilities = data['agent_capabilities'] max_tasks = int(data['max_tasks']) agent_location = data[ 'agent_location'] if 'agent_location' in data else None # Only protocol 1 supported for now if protocol != 1: raise HTTPError(400) try: session = create_session() except: logger.error("Failed to create database session for task request") raise HTTPError(500) try: query = session.query(Agent) agent = query.filter(Agent.uuid == agent_uuid).one() session.query(AgentCapability).filter( AgentCapability.agent_uuid == agent.uuid).delete() except NoResultFound: agent = Agent(uuid=agent_uuid, name=agent_name) session.add(agent) for agent_capability, agent_capability_info in agent_capabilities.items(): capability = AgentCapability(type=agent_capability, version=int( agent_capability_info['version']), agent=agent) session.add(capability) # Find all non-claimed tasks that the agent is able to handle and assign them to the agent query = session.query(AgentCapability, Task).filter(Task.assigned_agent_uuid.is_(None)).\ filter(AgentCapability.agent_uuid == agent.uuid).\ filter(and_(AgentCapability.type == Task.type, AgentCapability.version == Task.version)) tasks = [] # Assign available tasks to the agent and mark them as being in process for _, task in query[0:max_tasks]: task.assigned_agent_uuid = agent.uuid task.claimed = datetime.utcnow() tasks.append({ 'task_id': task.uuid, 'task_type': task.type, 'task_version': task.version, 'task_data': json.loads(task.data) }) agent.last_seen = datetime.utcnow() try: session.commit() except Exception: session.rollback() logger.error("Failed to commit database changes for task request") raise HTTPError(500) finally: session.close() logger.info( "Agent requested tasks - Agent's name and uuid: {}, {} - Agent was given following tasks: {}" .format(agent_name, agent_uuid, tasks)) return { "tasks": tasks, "return_time": (datetime.now(tz.tzlocal()) + timedelta(0, Settings.agent_return_time)).isoformat() }
def post_tasks(): data = request.json if data is None: logger.error("No JSON content in task response request!") raise HTTPError(400) try: jsonschema.validate(data, task_response_schema) except jsonschema.ValidationError as e: logger.error("Invalid JSON in task reponse: {0}".format(e)) raise HTTPError(400) protocol = int(data['protocol']) task_id = str(data['task_id']) # Only protocol 1 supported for now if protocol != 1: logger.error("Invalid protocol in task response: {0}".format(protocol)) raise HTTPError(400) try: session = create_session() except: logger.error("Failed to create database session for task result POST") raise HTTPError(500) try: task = session.query(Task).filter(Task.uuid == str(task_id)).one() if task.claimed is None or (task.completed is not None or task.failed is not None): logger.error("Incomplete task posted!") session.close() raise HTTPError(400) result = "" if 'task_data' in data: task.result_data = json.dumps(data['task_data']) task.completed = datetime.utcnow() result = json.dumps(task.result_data) elif 'task_error' in data: task.error = data['task_error'] task.failed = datetime.utcnow() result = task.error session.add(task) except NoResultFound: logger.error("No matching task in for task response!") session.close() raise HTTPError(400) try: session.commit() except Exception: session.rollback() logger.error("Failed to commit database changes for task result POST") raise HTTPError(500) finally: session.close() logger.info( "An agent returned task with results - uuid: {}, end results: {}". format(task_id, result))