def test_push_response():
test_app = TestApp(app)
session = create_session()
task = Task()
task.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546013'
task.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546013'
task.claimed = datetime.utcnow()
session.add(task)
session.commit()
test_app.post_json(
'/tasks/response', {
'protocol': 1,
'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'task_data': {
'key': 'value',
'another_key': 5
}
})
session = create_session()
task = session.query(Task).filter(
Task.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one()
assert task.completed is not None
assert task.result_data is not None
assert task.failed is None
assert task.error is None
task.completed = None
task.result_data = None
session.commit()
test_app.post_json(
'/tasks/response', {
'protocol': 1,
'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'task_error': 'Something went terribly wrong'
})
session = create_session()
task = session.query(Task).filter(
Task.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one()
assert task.completed is None
assert task.result_data is None
assert task.failed is not None
assert task.error is not None
session.close()
def dev_get_agents():
try:
session = create_session()
except:
raise HTTPError(500)
try:
query = session.query(Agent)
agents = query.all()
except Exception as e:
logger.exception(e)
session.close()
raise HTTPError(500)
agents_array = []
for agent in agents:
agent_json = {
'agent_id': agent.uuid,
'agent_name': agent.name,
'last_seen': str(agent.last_seen)
}
if agent.capabilities:
agent_json['agent_capabilities'] = {}
for capability in agent.capabilities:
agent_json['agent_capabilities'][capability.type] = {
'version': capability.version
}
agents_array.append(agent_json)
session.close()
return {'agents': agents_array}
def test_pull_task():
test_app = TestApp(app)
session = create_session()
task1 = Task()
task1.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546018'
task1.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546018'
task1.claimed = datetime.utcnow()
task1.data = json.dumps({'wait_time': 123})
session.add(task1)
task2 = Task()
task2.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546019'
task2.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546019'
task2.claimed = datetime.utcnow()
task2.completed = datetime.utcnow()
task2.result_data = json.dumps({'result': 'epic success'})
session.add(task2)
task3 = Task()
task3.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546020'
task3.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546020'
task3.claimed = datetime.utcnow()
task3.failed = datetime.utcnow()
task3.error = 'unknown error'
session.add(task3)
session.commit()
test_app.get('/task/de305d54-75b4-431b-adb2-eb6b9e546018')
test_app.get('/task/de305d54-75b4-431b-adb2-eb6b9e546019')
test_app.get('/task/de305d54-75b4-431b-adb2-eb6b9e546020')
def dev_get_agents():
try:
session = create_session()
except:
raise HTTPError(500)
try:
query = session.query(Agent)
agents = query.all()
except Exception as e:
logger.exception(e)
session.close()
raise HTTPError(500)
agents_array = []
for agent in agents:
agent_json = {
'agent_id': agent.uuid,
'agent_name': agent.name,
'last_seen': str(agent.last_seen)
}
if agent.capabilities:
agent_json['agent_capabilities'] = {}
for capability in agent.capabilities:
agent_json['agent_capabilities'][capability.type] = {
'version': capability.version
}
agents_array.append(agent_json)
session.close()
return {'agents': agents_array}
def status():
"""
Simple status page which verifies that database connectivity works and tells stats about pending tasks and active
agents
:return: dict in following format
{
'agents': 5, # Number of agents that have been active within last Settings.agent_active_threshold seconds
'tasks_waiting': 10 # Number of tasks that haven't been claimed yet
}
"""
try:
session = create_session()
except:
raise HTTPError(500, 'Something wen\'t wrong with database session')
agent_time_threshold = datetime.utcnow() - timedelta(0, Settings.agent_active_threshold)
try:
num_agents = session.query(Agent).filter(Agent.last_seen > agent_time_threshold).count()
tasks_waiting = session.query(Task).filter(Task.claimed is None).count()
except Exception as e:
raise HTTPError(500, 'Failed to query tasks and agents ' + str(e))
finally:
session.close()
return {'agents': num_agents, 'tasks_waiting': tasks_waiting}
def test_push_response():
test_app = TestApp(app)
session = create_session()
task = Task()
task.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546013'
task.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546013'
task.claimed = datetime.utcnow()
session.add(task)
session.commit()
test_app.post_json('/tasks/response', {
'protocol': 1,
'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'task_data': {
'key': 'value',
'another_key': 5
}
})
session = create_session()
task = session.query(Task).filter(Task.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one()
assert task.completed is not None
assert task.result_data is not None
assert task.failed is None
assert task.error is None
task.completed = None
task.result_data = None
session.commit()
test_app.post_json('/tasks/response', {
'protocol': 1,
'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'task_error': 'Something went terribly wrong'
})
session = create_session()
task = session.query(Task).filter(Task.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one()
assert task.completed is None
assert task.result_data is None
assert task.failed is not None
assert task.error is not None
session.close()
def test_poll_task_capability_change():
test_app = TestApp(app)
test_app.post_json('/tasks', {
'protocol': 1,
'agent_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'agent_name': 'Agent 007',
'agent_location': {
'country': 'FI',
'region': '18'
},
'agent_time': '2012-04-23T18:25:43.511Z',
'agent_capabilities': {
'task-type-1': {'version': 1},
'task-type-2': {'version': 2}
},
'max_tasks': 5
})
test_app.post_json('/tasks', {
'protocol': 1,
'agent_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'agent_name': 'Agent 007',
'agent_location': {
'country': 'FI',
'region': '18'
},
'agent_time': '2012-04-23T18:25:43.511Z',
'agent_capabilities': {
'task-type-1': {'version': 1},
'task-type-2': {'version': 2},
'task-type-3': {'version': 3},
'task-type-4': {'version': 4}
},
'max_tasks': 5
})
session = create_session()
agent = session.query(Agent).filter(Agent.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one()
try:
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-1').one()
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-2').one()
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-3').one()
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-4').one()
finally:
session.close()
def get_task(task_uuid):
"""
Gets information about single task with uuid task_uuid
:param task_uuid: uuid of the task
:return: dict in following format
{
'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', # UUID of the task (str)
'test_id': 'de305d54-75b4-431b-adb2-eb6b9e546013', # UUID of the test (str)
'task_type': 'wait', # type of the task (str)
'task_version': 1, # Version number of the task
'task_data': {}, # Dict containing data passed to the task (if any)
'task_completed': '31-03-2015:12:12:12', # Time when task was completed (if completed)
'task_result': {}, # Dict containing task's results (if completed)
'task_failed': '31-03-2015:12:12:12', # Time when task failed (if failed)
'task_error': 'Something went wrong' # Error that caused task to fail (if failed)
}
"""
try:
session = create_session()
except:
raise HTTPError(500)
try:
query = session.query(Task)
task = query.filter(Task.uuid == str(task_uuid)).one()
except NoResultFound:
raise HTTPError(404)
finally:
session.close()
task_desc = {
'task_id': task.uuid,
'test_id': task.test_id,
'task_type': task.type,
'task_version': task.version
}
if task.data is not None:
task_desc['task_data'] = json.loads(task.data)
if task.failed:
task_desc['task_failed'] = str(task.failed)
task_desc['task_error'] = str(task.error)
elif task.completed:
task_desc['task_completed'] = str(task.completed)
task_desc['task_result'] = json.loads(task.result_data)
return task_desc
def post_task():
data = request.json
if data is None:
raise HTTPError(400)
try:
jsonschema.validate(data, post_task_schema)
except jsonschema.ValidationError:
raise HTTPError(400)
session = create_session()
task_uuid = str(data['task_id'])
task_type = str(data['task_type'])
task_test_id = (data['test_id'])
task_data = ""
task = Task(
uuid=task_uuid,
type=task_type,
version=int(data['task_version']),
test_id=task_test_id
)
if 'task_data' in data:
task_data = json.dumps(data['task_data'])
task.data = task_data
try:
session.add(task)
except IntegrityError:
session.rollback()
raise HTTPError(400)
try:
session.commit()
except (IntegrityError, ProgrammingError):
session.rollback()
logger.error("Failed to commit database changes for BPMS task POST")
raise HTTPError(400)
finally:
session.close()
logger.info("Task posted by BPMS - Task's type: {}, test process id: {}, uuid: {}, parameters: {}"
.format(task_type, task_test_id, task_uuid, task_data))
def test_post_task_duplicate():
test_app = TestApp(app)
session = create_session()
task1 = Task()
task1.uuid = 'de305d54-75b4-431b-adb2-eb6b9e546018'
task1.test_id = 'de305d54-75b4-431b-adb2-eb6b9e546018'
task1.claimed = datetime.utcnow()
task1.data = json.dumps({'wait_time': 123})
session.add(task1)
session.commit()
session.close()
assert test_app.post_json('/task', {
'task_id': 'de305d54-75b4-431b-adb2-eb6b9e546018',
'test_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'task_type': 'wait',
'task_version': 1
}, expect_errors=True).status_int == 400
def dev_post_tasks():
data = request.json
if data is None:
raise HTTPError(400)
try:
jsonschema.validate(data, dev_post_tasks_schema)
except jsonschema.ValidationError:
raise HTTPError(400)
try:
session = create_session()
except:
raise HTTPError(500)
task = Task(
uuid=str(data['task_id']),
type=data['task_type'],
version=data['task_version'],
test_id=data['test_id'])
if 'task_data' in data:
task.data = json.dumps(data['task_data'])
try:
session.add(task)
except IntegrityError:
session.rollback()
session.close()
raise HTTPError(400)
try:
session.commit()
except (IntegrityError, ProgrammingError):
session.rollback()
raise HTTPError(400)
finally:
session.close()
def dev_post_tasks():
data = request.json
if data is None:
raise HTTPError(400)
try:
jsonschema.validate(data, dev_post_tasks_schema)
except jsonschema.ValidationError:
raise HTTPError(400)
try:
session = create_session()
except:
raise HTTPError(500)
task = Task(uuid=str(data['task_id']),
type=data['task_type'],
version=data['task_version'],
test_id=data['test_id'])
if 'task_data' in data:
task.data = json.dumps(data['task_data'])
try:
session.add(task)
except IntegrityError:
session.rollback()
session.close()
raise HTTPError(400)
try:
session.commit()
except (IntegrityError, ProgrammingError):
session.rollback()
raise HTTPError(400)
finally:
session.close()
def dev_get_tasks():
try:
session = create_session()
except:
raise HTTPError(500)
try:
query = session.query(Task)
tasks = query.all()
except Exception as e:
logger.exception(e)
raise HTTPError(500)
finally:
session.close()
tasks_array = []
for task in tasks:
task_desc = {
'task_id': task.uuid,
'task_type': task.type,
'task_version': task.version,
'test_id': task.test_id
}
if task.data is not None:
task_desc['task_data'] = json.loads(task.data)
if task.failed:
task_desc['task_failed'] = str(task.failed)
task_desc['task_error'] = str(task.error)
elif task.completed:
task_desc['task_completed'] = str(task.completed)
task_desc['task_result'] = str(task.result_data)
tasks_array.append(task_desc)
return {'tasks': tasks_array}
def dev_get_tasks():
try:
session = create_session()
except:
raise HTTPError(500)
try:
query = session.query(Task)
tasks = query.all()
except Exception as e:
logger.exception(e)
raise HTTPError(500)
finally:
session.close()
tasks_array = []
for task in tasks:
task_desc = {
'task_id': task.uuid,
'task_type': task.type,
'task_version': task.version,
'test_id': task.test_id
}
if task.data is not None:
task_desc['task_data'] = json.loads(task.data)
if task.failed:
task_desc['task_failed'] = str(task.failed)
task_desc['task_error'] = str(task.error)
elif task.completed:
task_desc['task_completed'] = str(task.completed)
task_desc['task_result'] = str(task.result_data)
tasks_array.append(task_desc)
return {'tasks': tasks_array}
def request_tasks():
data = request.json
if data is None:
raise HTTPError(400)
try:
jsonschema.validate(data, task_request_schema)
except jsonschema.ValidationError:
raise HTTPError(400)
protocol = int(data['protocol'])
agent_uuid = str(data['agent_id'])
agent_name = str(data['agent_name'])
agent_time = data['agent_time']
agent_capabilities = data['agent_capabilities']
max_tasks = int(data['max_tasks'])
agent_location = data[
'agent_location'] if 'agent_location' in data else None
# Only protocol 1 supported for now
if protocol != 1:
raise HTTPError(400)
try:
session = create_session()
except:
logger.error("Failed to create database session for task request")
raise HTTPError(500)
try:
query = session.query(Agent)
agent = query.filter(Agent.uuid == agent_uuid).one()
session.query(AgentCapability).filter(
AgentCapability.agent_uuid == agent.uuid).delete()
except NoResultFound:
agent = Agent(uuid=agent_uuid, name=agent_name)
session.add(agent)
for agent_capability, agent_capability_info in agent_capabilities.items():
capability = AgentCapability(type=agent_capability,
version=int(
agent_capability_info['version']),
agent=agent)
session.add(capability)
# Find all non-claimed tasks that the agent is able to handle and assign them to the agent
query = session.query(AgentCapability, Task).filter(Task.assigned_agent_uuid.is_(None)).\
filter(AgentCapability.agent_uuid == agent.uuid).\
filter(and_(AgentCapability.type == Task.type, AgentCapability.version == Task.version))
tasks = []
# Assign available tasks to the agent and mark them as being in process
for _, task in query[0:max_tasks]:
task.assigned_agent_uuid = agent.uuid
task.claimed = datetime.utcnow()
tasks.append({
'task_id': task.uuid,
'task_type': task.type,
'task_version': task.version,
'task_data': json.loads(task.data)
})
agent.last_seen = datetime.utcnow()
try:
session.commit()
except Exception:
session.rollback()
logger.error("Failed to commit database changes for task request")
raise HTTPError(500)
finally:
session.close()
logger.info(
"Agent requested tasks - Agent's name and uuid: {}, {} - Agent was given following tasks: {}"
.format(agent_name, agent_uuid, tasks))
return {
"tasks":
tasks,
"return_time": (datetime.now(tz.tzlocal()) +
timedelta(0, Settings.agent_return_time)).isoformat()
}
def post_tasks():
data = request.json
if data is None:
logger.error("No JSON content in task response request!")
raise HTTPError(400)
try:
jsonschema.validate(data, task_response_schema)
except jsonschema.ValidationError as e:
logger.error("Invalid JSON in task reponse: {0}".format(e))
raise HTTPError(400)
protocol = int(data['protocol'])
task_id = str(data['task_id'])
# Only protocol 1 supported for now
if protocol != 1:
logger.error("Invalid protocol in task response: {0}".format(protocol))
raise HTTPError(400)
try:
session = create_session()
except:
logger.error("Failed to create database session for task result POST")
raise HTTPError(500)
try:
task = session.query(Task).filter(Task.uuid == str(task_id)).one()
if task.claimed is None or (task.completed is not None
or task.failed is not None):
logger.error("Incomplete task posted!")
session.close()
raise HTTPError(400)
result = ""
if 'task_data' in data:
task.result_data = json.dumps(data['task_data'])
task.completed = datetime.utcnow()
result = json.dumps(task.result_data)
elif 'task_error' in data:
task.error = data['task_error']
task.failed = datetime.utcnow()
result = task.error
session.add(task)
except NoResultFound:
logger.error("No matching task in for task response!")
session.close()
raise HTTPError(400)
try:
session.commit()
except Exception:
session.rollback()
logger.error("Failed to commit database changes for task result POST")
raise HTTPError(500)
finally:
session.close()
logger.info(
"An agent returned task with results - uuid: {}, end results: {}".
format(task_id, result))
def post_tasks():
data = request.json
if data is None:
logger.error("No JSON content in task response request!")
raise HTTPError(400)
try:
jsonschema.validate(data, task_response_schema)
except jsonschema.ValidationError as e:
logger.error("Invalid JSON in task reponse: {0}".format(e))
raise HTTPError(400)
protocol = int(data['protocol'])
task_id = str(data['task_id'])
# Only protocol 1 supported for now
if protocol != 1:
logger.error("Invalid protocol in task response: {0}".format(protocol))
raise HTTPError(400)
try:
session = create_session()
except:
logger.error("Failed to create database session for task result POST")
raise HTTPError(500)
try:
task = session.query(Task).filter(Task.uuid == str(task_id)).one()
if task.claimed is None or (task.completed is not None or task.failed is not None):
logger.error("Incomplete task posted!")
session.close()
raise HTTPError(400)
result = ""
if 'task_data' in data:
task.result_data = json.dumps(data['task_data'])
task.completed = datetime.utcnow()
result = json.dumps(task.result_data)
elif 'task_error' in data:
task.error = data['task_error']
task.failed = datetime.utcnow()
result = task.error
session.add(task)
except NoResultFound:
logger.error("No matching task in for task response!")
session.close()
raise HTTPError(400)
try:
session.commit()
except Exception:
session.rollback()
logger.error("Failed to commit database changes for task result POST")
raise HTTPError(500)
finally:
session.close()
logger.info("An agent returned task with results - uuid: {}, end results: {}".format(task_id, result))
def request_tasks():
data = request.json
if data is None:
raise HTTPError(400)
try:
jsonschema.validate(data, task_request_schema)
except jsonschema.ValidationError:
raise HTTPError(400)
protocol = int(data['protocol'])
agent_uuid = str(data['agent_id'])
agent_name = str(data['agent_name'])
agent_time = data['agent_time']
agent_capabilities = data['agent_capabilities']
max_tasks = int(data['max_tasks'])
agent_location = data['agent_location'] if 'agent_location' in data else None
# Only protocol 1 supported for now
if protocol != 1:
raise HTTPError(400)
try:
session = create_session()
except:
logger.error("Failed to create database session for task request")
raise HTTPError(500)
try:
query = session.query(Agent)
agent = query.filter(Agent.uuid == agent_uuid).one()
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).delete()
except NoResultFound:
agent = Agent(uuid=agent_uuid, name=agent_name)
session.add(agent)
for agent_capability, agent_capability_info in agent_capabilities.items():
capability = AgentCapability(type=agent_capability,
version=int(agent_capability_info['version']),
agent=agent)
session.add(capability)
# Find all non-claimed tasks that the agent is able to handle and assign them to the agent
query = session.query(AgentCapability, Task).filter(Task.assigned_agent_uuid.is_(None)).\
filter(AgentCapability.agent_uuid == agent.uuid).\
filter(and_(AgentCapability.type == Task.type, AgentCapability.version == Task.version))
tasks = []
# Assign available tasks to the agent and mark them as being in process
for _, task in query[0:max_tasks]:
task.assigned_agent_uuid = agent.uuid
task.claimed = datetime.utcnow()
tasks.append({
'task_id': task.uuid,
'task_type': task.type,
'task_version': task.version,
'task_data': json.loads(task.data)
})
agent.last_seen = datetime.utcnow()
try:
session.commit()
except Exception:
session.rollback()
logger.error("Failed to commit database changes for task request")
raise HTTPError(500)
finally:
session.close()
logger.info("Agent requested tasks - Agent's name and uuid: {}, {} - Agent was given following tasks: {}"
.format(agent_name, agent_uuid, tasks))
return {
"tasks": tasks,
"return_time": (datetime.now(tz.tzlocal()) + timedelta(0, Settings.agent_return_time)).isoformat()
}
def test_poll_task_capability_change():
test_app = TestApp(app)
test_app.post_json(
'/tasks', {
'protocol': 1,
'agent_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'agent_name': 'Agent 007',
'agent_location': {
'country': 'FI',
'region': '18'
},
'agent_time': '2012-04-23T18:25:43.511Z',
'agent_capabilities': {
'task-type-1': {
'version': 1
},
'task-type-2': {
'version': 2
}
},
'max_tasks': 5
})
test_app.post_json(
'/tasks', {
'protocol': 1,
'agent_id': 'de305d54-75b4-431b-adb2-eb6b9e546013',
'agent_name': 'Agent 007',
'agent_location': {
'country': 'FI',
'region': '18'
},
'agent_time': '2012-04-23T18:25:43.511Z',
'agent_capabilities': {
'task-type-1': {
'version': 1
},
'task-type-2': {
'version': 2
},
'task-type-3': {
'version': 3
},
'task-type-4': {
'version': 4
}
},
'max_tasks': 5
})
session = create_session()
agent = session.query(Agent).filter(
Agent.uuid == 'de305d54-75b4-431b-adb2-eb6b9e546013').one()
try:
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-1').one()
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-2').one()
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-3').one()
session.query(AgentCapability).filter(AgentCapability.agent_uuid == agent.uuid).\
filter(AgentCapability.type == 'task-type-4').one()
finally:
session.close()