def cancel_test():
"""Cancel a scheduled test"""
test_id = request.form['test_id']
test = db.get_test(test_id)
# If test is scheduled, we can immediately cancel.
# If test is already in progress, we need to mark as
# cancel_pending to await the client to cancel the job itself.
new_status = 'cancelled'
if test['status'] == 'in_progress' or test['status'] == 'cancel_pending':
new_status = 'cancel_pending'
if user_in_role('admin'):
db.update_test_status(test_id, new_status)
else:
# Check if the test is owned by the user:
if test['user'] == get_user_id():
db.update_test_status(test_id, new_status)
else:
return make_response(
jsonify({
'error':
'Access Denied to modify test {test_id}'.format(
test_id=test_id)
}), 401)
return jsonify({'success': 'Test cancelled'})
def test_done(command):
"""Receive completed test artifacts from client"""
db.update_test_status(command['test_id'], command['status'])
# Record test failure message, if any:
if command['status'] == 'failed':
msg = (command.get('message','') + "\n" + command.get('stacktrace','')).strip()
db.update_test_artifact(command['test_id'], 'failure', msg)
# Send response:
command.respond(test_id=command['test_id'], message='test_update', done=True)
def test_done(command):
"""Receive completed test artifacts from client"""
db.update_test_status(command['test_id'], command['status'])
# Record test failure message, if any:
if command['status'] == 'failed':
msg = (command.get('message', '') + "\n" +
command.get('stacktrace', '')).strip()
db.update_test_artifact(command['test_id'], 'failure', msg)
# Send response:
command.respond(test_id=command['test_id'],
message='test_update',
done=True)
def get_work(command):
# Mark any existing in_process jobs for this cluster as
# failed. If the cluster is asking for new work, then these
# got dropped on the floor:
for test in db.get_in_progress_tests(context['cluster']):
db.update_test_status(test['test_id'], 'failed')
# Find the next test scheduled for the client's cluster:
tests = db.get_scheduled_tests(context['cluster'], limit=1)
if len(tests) > 0:
test_id = tests[0]['test_id']
else:
# No tests are currently scheduled.
# Register a zmq listener of notifications of incoming tests, with a timeout.
# When we see any test scheduled notification for our cluster, redo the query.
# If timeout reached, redo the query anyway in case we missed the notification.
def setup_zmq():
zmq_context = zmq.Context()
zmq_socket = zmq_context.socket(zmq.SUB)
zmq_socket.connect('tcp://127.0.0.1:5557')
zmq_socket.setsockopt_string(
zmq.SUBSCRIBE,
unicode('scheduled {cluster} '.format(
cluster=context['cluster'])))
zmq_socket.setsockopt(zmq.RCVTIMEO, 15000)
return zmq_socket
zmq_socket = setup_zmq()
while True:
try:
cluster, test_id = zmq_socket.recv_string().split()
except zmq.error.Again:
pass
except zmq.error.ZMQError, e:
if e.errno == zmq.POLLERR:
log.error(e)
# Interrupted zmq socket code, reinitialize:
# I get this when I resize my terminal.. WTF?
zmq_socket = setup_zmq()
finally:
tests = db.get_scheduled_tests(context['cluster'], limit=1)
def cancel_test():
"""Cancel a scheduled test"""
test_id = request.form['test_id']
test = db.get_test(test_id)
# If test is scheduled, we can immediately cancel.
# If test is already in progress, we need to mark as
# cancel_pending to await the client to cancel the job itself.
new_status = 'cancelled'
if test['status'] == 'in_progress' or test['status'] == 'cancel_pending':
new_status = 'cancel_pending'
if user_in_role('admin'):
db.update_test_status(test_id, new_status)
else:
# Check if the test is owned by the user:
if test['user'] == get_user_id():
db.update_test_status(test_id, new_status)
else:
return make_response(jsonify({'error':'Access Denied to modify test {test_id}'
.format(test_id=test_id)}), 401)
return jsonify({'success':'Test cancelled'})
def get_work(command):
# Mark any existing in_process jobs for this cluster as
# failed. If the cluster is asking for new work, then these
# got dropped on the floor:
for test in db.get_in_progress_tests(context['cluster']):
db.update_test_status(test['test_id'], 'failed')
# Find the next test scheduled for the client's cluster:
tests = db.get_scheduled_tests(context['cluster'], limit=1)
if len(tests) > 0:
test_id = tests[0]['test_id']
else:
# No tests are currently scheduled.
# Register a zmq listener of notifications of incoming tests, with a timeout.
# When we see any test scheduled notification for our cluster, redo the query.
# If timeout reached, redo the query anyway in case we missed the notification.
def setup_zmq():
zmq_context = zmq.Context()
zmq_socket = zmq_context.socket(zmq.SUB)
zmq_socket.connect('tcp://127.0.0.1:5557')
zmq_socket.setsockopt_string(
zmq.SUBSCRIBE,
unicode('scheduled {cluster} '.format(cluster=context['cluster'])))
zmq_socket.setsockopt(zmq.RCVTIMEO, 15000)
return zmq_socket
zmq_socket = setup_zmq()
while True:
try:
cluster, test_id = zmq_socket.recv_string().split()
except zmq.error.Again:
pass
except zmq.error.ZMQError, e:
if e.errno == zmq.POLLERR:
log.error(e)
# Interrupted zmq socket code, reinitialize:
# I get this when I resize my terminal.. WTF?
zmq_socket = setup_zmq()
finally:
tests = db.get_scheduled_tests(context['cluster'], limit=1)
def cancel_test():
"""Cancel a scheduled test"""
test_id = request.form["test_id"]
test = db.get_test(test_id)
# If test is scheduled, we can immediately cancel.
# If test is already in progress, we need to mark as
# cancel_pending to await the client to cancel the job itself.
new_status = "cancelled"
if test["status"] == "in_progress" or test["status"] == "cancel_pending":
new_status = "cancel_pending"
if user_in_role("admin"):
db.update_test_status(test_id, new_status)
else:
# Check if the test is owned by the user:
if test["user"] == get_user_id():
db.update_test_status(test_id, new_status)
else:
return make_response(
jsonify({"error": "Access Denied to modify test {test_id}".format(test_id=test_id)}), 401
)
return jsonify({"success": "Test cancelled"})
tests = db.get_scheduled_tests(context['cluster'], limit=1)
if len(tests) > 0:
test_id = tests[0]['test_id']
break
else:
# Send no-work-yet message:
console_publish(context['cluster'], {'ctl':'WAIT'})
command.respond(action='wait', follow_up=False)
test = db.get_test(test_id)
# Give the test to the client:
response = command.respond(test=test)
# Expect an prepared status message back:
assert response['test_id'] == test['test_id'] and \
response['status'] == 'prepared'
# Update the test status:
db.update_test_status(test['test_id'], 'in_progress')
# Let the client know they can start it:
response.respond(test_id=test['test_id'], status="in_progress", done=True)
def test_done(command):
"""Receive completed test artifacts from client"""
db.update_test_status(command['test_id'], command['status'])
# Record test failure message, if any:
if command['status'] == 'failed':
msg = (command.get('message','') + "\n" + command.get('stacktrace','')).strip()
db.update_test_artifact(command['test_id'], 'failure', msg)
# Send response:
command.respond(test_id=command['test_id'], message='test_update', done=True)
def receive_artifact_chunk_object(command):
command.respond(message="ready", follow_up=False, done=False)
tests = db.get_scheduled_tests(context['cluster'], limit=1)
if len(tests) > 0:
test_id = tests[0]['test_id']
break
else:
# Send no-work-yet message:
console_publish(context['cluster'], {'ctl': 'WAIT'})
command.respond(action='wait', follow_up=False)
test = db.get_test(test_id)
# Give the test to the client:
response = command.respond(test=test)
# Expect an prepared status message back:
assert response['test_id'] == test['test_id'] and \
response['status'] == 'prepared'
# Update the test status:
db.update_test_status(test['test_id'], 'in_progress')
# Let the client know they can start it:
response.respond(test_id=test['test_id'],
status="in_progress",
done=True)
def test_done(command):
"""Receive completed test artifacts from client"""
db.update_test_status(command['test_id'], command['status'])
# Record test failure message, if any:
if command['status'] == 'failed':
msg = (command.get('message', '') + "\n" +
command.get('stacktrace', '')).strip()
db.update_test_artifact(command['test_id'], 'failure', msg)
# Send response:
command.respond(test_id=command['test_id'],