def tests_name_spec(name):
cursor = dbcursor_query("SELECT EXISTS (SELECT * FROM test"
" WHERE available AND name = %s)",
[ name ])
exists = cursor.fetchone()[0]
cursor.close()
if not exists:
return not_found()
try:
args = arg_json('args')
except ValueError as ex:
return bad_request("JSON passed to 'args': %s " % (str(ex)))
status, stdout, stderr = pscheduler.run_program(
[ 'pscheduler', 'internal', 'invoke', 'test', name, 'cli-to-spec' ],
stdin = pscheduler.json_dump(args),
timeout=5
)
if status != 0:
return bad_request(stderr)
# The extra parse here makes 'pretty' work.
returned_json = pscheduler.json_load(stdout)
return ok_json(returned_json, sanitize=False)
def __init__(
self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.test = data['test']
self.limit = data['limit']
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test", self.test,
"limit-is-valid"
],
stdin=pscheduler.json_dump(self.limit),
# TODO: Is this reasonable?
timeout=5)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
result = pscheduler.json_load(stdout, max_schema=1)
if not result['valid']:
raise ValueError("Invalid limit: %s" % result['message'])
def tests_name_spec_is_valid(name):
cursor = dbcursor_query(
"SELECT EXISTS"
" (SELECT * FROM test WHERE available AND name = %s)",
[name])
exists = cursor.fetchone()[0]
cursor.close()
if not exists:
return not_found()
spec = request.args.get('spec')
if spec is None:
return bad_request("No test spec provided")
try:
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test",
name, "spec-is-valid"],
stdin=spec)
if returncode != 0:
return error("Unable to validate test spec: %s" % (stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
return ok_json(validate_json)
except Exception as ex:
return error("Unable to validate test spec: %s" % (str(ex)))
def run(
self,
# These are lifted straight from run_program.
line_call=None, # Lambda to call when a line arrives
timeout=None, # Seconds
timeout_ok=False, # Treat timeouts as not being an error
fail_message=None, # Exit with this failure message
env=None, # Environment for new process, None=existing
env_add=None, # Add hash to existing environment
attempts=10): # Max attempts to start the process
"""
Run the chain. Return semantics are the same as for
pscheduler.run_program(): a tuple of status, stdout, stderr.
"""
# TODO: Is there a more-pythonic way to do this than pasting
# in all of the args?
result = pscheduler.run_program([self.stages[0]],
line_call=line_call,
timeout=timeout,
timeout_ok=timeout_ok,
fail_message=fail_message,
env=env,
env_add=env_add,
attempts=attempts)
for remove in self.stages:
try:
pass # os.unlink(remove)
except IOError:
pass # This is best effort only.
return result
def tests_name_spec(name):
try:
cursor = dbcursor_query("SELECT EXISTS (SELECT * FROM test WHERE NAME = %s)",
[ name ])
except Exception as ex:
return error(str(ex))
exists = cursor.fetchone()[0]
if not exists:
return not_found()
try:
args = arg_json('args')
except ValueError:
return error("Invalid JSON passed to 'args'")
status, stdout, stderr = pscheduler.run_program(
[ 'pscheduler', 'internal', 'invoke', 'test', name, 'cli-to-spec' ],
stdin = pscheduler.json_dump(args),
short = True,
)
if status != 0:
return bad_request(stderr)
# The extra parse here makes 'pretty' work.
returned_json = pscheduler.json_load(stdout)
return ok_json(returned_json)
def evaluate(self, run): # The proposed run
# Dissent if the test isn't our type
if run["type"] != self.test:
return {"passed": False, "reasons": ["Test is not '%s'" % self.test]}
pass_input = {"spec": run["spec"], "limit": self.limit}
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", self.test, "limit-passes"],
stdin=pscheduler.json_dump(pass_input),
# TODO: Is this reasonable?
timeout=5,
)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
check_result = pscheduler.json_load(stdout)
passed = check_result["passes"]
result = {"passed": passed}
if not passed:
result["reasons"] = check_result["errors"]
return result
def tests_name_participants(name):
spec = request.args.get('spec')
if spec is None:
return bad_request("No test spec provided")
# HACK: BWCTLBC -- BEGIN
lead_bind = request.args.get("lead-bind")
if lead_bind is not None:
env_add = {"PSCHEDULER_LEAD_BIND_HACK": lead_bind}
else:
env_add = None
# HACK: BWCTLBC --- END
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", name,
"participants"],
stdin = spec,
env_add=env_add # HACK: BWCTLBC
)
except KeyError:
return bad_request("Invalid spec")
except Exception as ex:
return bad_request(ex)
if returncode != 0:
return bad_request(stderr)
# If this fails because of bad JSON, an exception will be thrown,
# caught and logged.
return json_response(pscheduler.json_load(stdout, max_schema=1))
def evaluate(
self,
run # The proposed run
):
# Dissent if the test isn't our type
if run["type"] != self.test:
return {
"passed": False,
"reasons": ["Test is not '%s'" % self.test]
}
pass_input = {"spec": run["spec"], "limit": self.limit}
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test", self.test,
"limit-passes"
],
stdin=pscheduler.json_dump(pass_input),
# TODO: Is this reasonable?
timeout=5)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
check_result = pscheduler.json_load(stdout, max_schema=1)
passed = check_result["passes"]
result = {"passed": passed, "limit": self.limit}
if not passed:
result["reasons"] = check_result["errors"]
return result
def contexts_name_data_is_valid(name):
try:
cursor = dbcursor_query(
"SELECT EXISTS"
" (SELECT * FROM context WHERE available AND NAME = %s)",
[name])
except Exception as ex:
return error(str(ex))
exists = cursor.fetchone()[0]
cursor.close()
if not exists:
return not_found()
data = request.args.get('data')
if data is None:
return bad_request("No archive data provided")
try:
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "context",
name, "data-is-valid"],
stdin=data)
if returncode != 0:
return error("Unable to validate context data: %s" % (stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
return ok_json(validate_json)
except Exception as ex:
return error("Unable to validate context data: %s" % (str(ex)))
def run_cmd(self,
input,
args=[],
expected_status=0,
json_out=True,
expected_stderr=""):
"""
Run and verify results. Takes following params:
Args:
input: a string to be fed via stdin to the program being executed
args: optional command-line arguments to be given to the command-run.
expected_status: the expected return code of the program. default is 0.
json_out: indicates whether output should be expected to be valid JSON
expected_stderr: string to match against stderr
Returns:
stdout as json dict if json_out is True (default), otehrwise a string
"""
#Run command
full_cmd = [self.cmd] + args
try:
status, stdout, stderr = run_program(full_cmd, stdin=input)
print self.cmd
except:
#print stacktrace for any errors
traceback.print_exc()
self.fail("unable to run command {0}".format(self.cmd))
#check stdout and stderr
print stdout
print stderr
self.assertEqual(status, expected_status) #status should be 0
if expected_status == 0:
self.assertFalse(stderr) #stderr should be empty
elif expected_stderr:
self.assertEquals(stderr, expected_stderr)
if not json_out:
return stdout
#check for valid JSON
try:
result_json = json.loads(stdout)
except:
traceback.print_exc()
self.fail("Invalid JSON returned by {0}: {1}".format(
self.progname, stdout))
return result_json
def get_output(self, args, check_success=True):
args.insert(0, "%s/../cli-to-spec" % self.path)
# actually run cli-to-spec with the input
code, stdout, stderr = pscheduler.run_program(args)
if check_success:
# make sure it succeeded
self.assertEqual(code, 0)
# get json out
if code != 0:
return stderr
return json.loads(stdout)
def get_output(self, limit, spec, check_success=True):
args = {"limit": limit, "spec": spec}
# actually run cli-to-spec with the input
code, stdout, stderr = pscheduler.run_program("%s/../limit-passes" %
self.path,
stdin=json.dumps(args))
if check_success:
# make sure it succeeded
self.assertEqual(code, 0)
# get json out
if code != 0:
return stderr
return json.loads(stdout)
def _run_cmd(self,
input,
expected_status=0,
expected_valid=True,
expected_errors=[],
match_all_errors=True):
#Run command
self._init_cmd()
try:
status, stdout, stderr = run_program([self.vaildate_cmd],
stdin=input)
except:
#print stacktrace for any errors
traceback.print_exc()
self.fail("unable to run limit-passes command {0}".format(
self.vaildate_cmd))
#check stdout and stderr
print stdout
print stderr
self.assertEqual(status, expected_status) #status should be 0
self.assertFalse(stderr) #stderr should be empty
#check for valid JSON
try:
result_json = json.loads(stdout)
except:
traceback.print_exc()
self.fail(
"Invalid JSON returned by limit-passes: {0}".format(stdout))
#check fields
assert ('passes' in result_json)
self.assertEqual(result_json['passes'], expected_valid)
if expected_valid:
assert ('errors' not in result_json)
else:
assert ('errors' in result_json)
if len(expected_errors) > 0 and match_all_errors:
#verify list of errors same length
self.assertEqual(len(result_json['errors']),
len(expected_errors))
for expected_error in expected_errors:
#verify messages are in list
assert (expected_error in result_json['errors'])
def tasks_uuid_cli(uuid):
if not uuid_is_valid(uuid):
return not_found()
# Get a task, adding server-derived details if a 'detail'
# argument is present.
try:
cursor = dbcursor_query(
"""SELECT
task.json #>> '{test, spec}',
test.name
FROM
task
JOIN test on test.id = task.test
WHERE task.uuid = %s""", [uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if row is None:
return not_found()
json, test = row
try:
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", test, "spec-to-cli"],
stdin=json)
if returncode != 0:
return error("Unable to convert test spec: " + stderr)
except Exception as ex:
return error("Unable to convert test spec: " + str(ex))
returned = pscheduler.json_load(stdout)
returned.insert(0, test)
return ok(pscheduler.json_dump(returned))
def __init__(self, data): # Data suitable for this class
valid, message = test_data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.test = data["test"]
self.limit = data["limit"]
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", self.test, "limit-is-valid"],
stdin=pscheduler.json_dump(self.limit),
# TODO: Is this reasonable?
timeout=5,
)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
result = pscheduler.json_load(stdout)
if not result["valid"]:
raise ValueError("Invalid limit: %s" % result["message"])
def tasks_uuid_cli(uuid):
# Get a task, adding server-derived details if a 'detail'
# argument is present.
try:
cursor = dbcursor_query(
"""SELECT
task.json #>> '{test, spec}',
test.name
FROM
task
JOIN test on test.id = task.test
WHERE task.uuid = %s""", [uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if row is None:
return not_found()
json, test = row
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
test, "spec-to-cli" ], stdin = json )
if returncode != 0:
return error("Unable to convert test spec: " + stderr)
except Exception as ex:
return error("Unable to convert test spec: " + str(ex))
returned = pscheduler.json_load(stdout)
returned.insert(0, test)
return ok(pscheduler.json_dump(returned))
def tests_name_participants(name):
spec = request.args.get('spec')
if spec is None:
return bad_request("No test spec provided")
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", name,
"participants"],
stdin = spec
)
except KeyError:
return bad_request("Invalid spec")
except Exception as ex:
return bad_request(ex)
if returncode != 0:
return bad_request(stderr)
# If this fails because of bad JSON, an exception will be thrown,
# caught and logged.
return json_response(pscheduler.json_load(stdout))
def tests_name_participants(name):
spec = request.args.get('spec')
if spec is None:
return bad_request("No test spec provided")
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", name,
"participants"],
stdin = spec,
)
except KeyError:
return bad_request("Invalid spec")
except Exception as ex:
return bad_request(ex)
if returncode != 0:
return bad_request(stderr)
# If this fails because of bad JSON, an exception will be thrown,
# caught and logged.
return json_response(pscheduler.json_load(stdout, max_schema=1))
def tests_name_lead(name):
spec = request.args.get('spec')
if spec is None:
return bad_request("No test spec provided")
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", name,
"participants"],
stdin = spec
)
except KeyError:
return bad_request("Invalid spec")
except Exception as ex:
return bad_request(ex)
if returncode != 0:
return bad_request(stderr)
part_list = pscheduler.json_load(stdout)
lead = part_list['participants'][0]
return json_response(lead)
def _run_cmd(self,
input,
expected_status=0,
expected_valid=True,
expected_error=""):
#Run command
try:
status, stdout, stderr = run_program([self.vaildate_cmd],
stdin=input)
except:
#print stacktrace for any errors
traceback.print_exc()
self.fail("unable to run limit-is-valid command {0}".format(
self.vaildate_cmd))
#check stdout and stderr
print stdout
print stderr
self.assertEqual(status, expected_status) #status should be 0
self.assertFalse(stderr) #stderr should be empty
#check for valid JSON
try:
result_json = json.loads(stdout)
except:
traceback.print_exc()
self.fail(
"Invalid JSON returned by limit-is-valid: {0}".format(stdout))
#check fields
assert (self.result_valid_field in result_json)
self.assertEqual(result_json[self.result_valid_field], expected_valid)
if expected_valid:
assert (self.error_field not in result_json)
else:
assert (self.error_field in result_json)
if expected_error:
self.assertEqual(result_json[self.error_field], expected_error)
def tasks_uuid_runs_run_result(task, run):
if not uuid_is_valid(task) or not uuid_is_valid(run):
return not_found()
wait = arg_boolean('wait')
format = request.args.get('format')
if format is None:
format = 'application/json'
if format not in ['application/json', 'text/html', 'text/plain']:
return bad_request("Unsupported format " + format)
# If asked for 'first', dig up the first run and use its UUID.
# This is more for debug convenience than anything else.
if run == 'first':
try:
run = __runs_first_run(task)
except Exception as ex:
log.exception()
return error(str(ex))
if run is None:
return not_found()
#
# Camp on the run for a result
#
# 40 tries at 0.25s intervals == 10 sec.
tries = 40 if wait else 1
while tries:
try:
cursor = dbcursor_query(
"""
SELECT
test.name,
run.result_merged,
task.json #> '{test, spec}'
FROM
run
JOIN task ON task.id = run.task
JOIN test ON test.id = task.test
WHERE
task.uuid = %s
AND run.uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
cursor.close()
return not_found()
# TODO: Make sure we got back one row with two columns.
row = cursor.fetchone()
cursor.close()
if not wait and row[1] is None:
time.sleep(0.25)
tries -= 1
else:
break
if tries == 0:
return not_found()
test_type, merged_result, test_spec = row
# JSON requires no formatting.
if format == 'application/json':
return ok_json(merged_result)
if not merged_result['succeeded']:
if format == 'text/plain':
return ok("Run failed.", mimetype=format)
elif format == 'text/html':
return ok("<p>Run failed.</p>", mimetype=format)
return bad_request("Unsupported format " + format)
formatter_input = {"spec": test_spec, "result": merged_result}
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test", test_type,
"result-format", format
],
stdin=pscheduler.json_dump(formatter_input))
if returncode != 0:
return error("Failed to format result: " + stderr)
return ok(stdout.rstrip(), mimetype=format)
def tasks():
if request.method == 'GET':
expanded = is_expanded()
query = """
SELECT json, uuid
FROM task
"""
args = []
try:
json_query = arg_json("json")
except ValueError as ex:
return bad_request(str(ex))
if json_query is not None:
query += "WHERE json @> %s"
args.append(request.args.get("json"))
query += " ORDER BY added"
try:
cursor = dbcursor_query(query, args)
except Exception as ex:
return error(str(ex))
result = []
for row in cursor:
url = base_url(row[1])
if not expanded:
result.append(url)
continue
row[0]['href'] = url
result.append(row[0])
return json_response(result)
elif request.method == 'POST':
try:
task = pscheduler.json_load(request.data)
except ValueError:
return bad_request("Invalid JSON:" + request.data)
# TODO: Validate the JSON against a TaskSpecification
# See if the task spec is valid
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
task['test']['type'], "spec-is-valid" ],
stdin = pscheduler.json_dump(task['test']['spec'])
)
if returncode != 0:
return error("Invalid test specification: " + stderr)
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
log.debug("Validated test: %s", pscheduler.json_dump(task['test']))
# Find the participants
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
task['test']['type'], "participants" ],
stdin = pscheduler.json_dump(task['test']['spec'])
)
if returncode != 0:
return error("Unable to determine participants: " + stderr)
participants = [ host if host is not None
else pscheduler.api_this_host()
for host in pscheduler.json_load(stdout)["participants"] ]
except Exception as ex:
return error("Unable to determine participants: " + str(ex))
nparticipants = len(participants)
# TODO: The participants must be unique. This should be
# verified by fetching the host name from each one.
#
# TOOL SELECTION
#
# TODO: Need to provide for tool being specified by the task
# package.
tools = []
for participant in participants:
try:
# TODO: This will fail with a very large test spec.
status, result = pscheduler.url_get(
pscheduler.api_url(participant, "tools"),
params={ 'test': pscheduler.json_dump(task['test']) }
)
if status != 200:
raise Exception("%d: %s" % (status, result))
tools.append(result)
except Exception as ex:
return error("Error getting tools from %s: %s" \
% (participant, str(ex)))
log.debug("Participant %s offers tools %s", participant, tools)
if len(tools) != nparticipants:
return error("Didn't get a full set of tool responses")
if "tools" in task:
tool = pick_tool(tools, pick_from=task['tools'])
else:
tool = pick_tool(tools)
if tool is None:
# TODO: This could stand some additional diagnostics.
return no_can_do("Couldn't find a tool in common among the participants.")
task['tool'] = tool
#
# TASK CREATION
#
task_data = pscheduler.json_dump(task)
log.debug("Task data: %s", task_data)
tasks_posted = []
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on %s", task["test"])
(processor, whynot) = limitprocessor()
if processor is None:
log.debug("Limit processor is not initialized. %s", whynot)
return no_can_do("Limit processor is not initialized: %s" % whynot)
# TODO: This is cooked up in two places. Make a function of it.
hints = {
"ip": request.remote_addr
}
hints_data = pscheduler.json_dump(hints)
log.debug("Processor = %s" % processor)
passed, diags = processor.process(task["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
# Post the lead with the local database, which also assigns
# its UUID. Make it disabled so the scheduler doesn't try to
# do anything with it until the task has been submitted to all
# of the other participants.
try:
cursor = dbcursor_query("SELECT * FROM api_task_post(%s, %s, 0, NULL, FALSE)",
[task_data, hints_data], onerow=True)
except Exception as ex:
return error(str(ex.diag.message_primary))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
task_uuid = cursor.fetchone()[0]
log.debug("Tasked lead, UUID %s", task_uuid)
# Other participants get the UUID forced upon them.
for participant in range(1,nparticipants):
part_name = participants[participant]
try:
log.debug("Tasking %d@%s: %s", participant, part_name, task_data)
post_url = pscheduler.api_url(part_name,
'tasks/' + task_uuid)
log.debug("Posting task to %s", post_url)
status, result = pscheduler.url_post(
post_url,
params={ 'participant': participant },
data=task_data,
json=False,
throw=False)
log.debug("Remote returned %d: %s", status, result)
if status != 200:
raise Exception("Unable to post task to %s: %s"
% (part_name, result))
tasks_posted.append(result)
except Exception as ex:
log.exception()
for url in tasks_posted:
# TODO: Handle failure?
status, result = requests.delete(url)
try:
dbcursor_query("SELECT api_task_delete(%s)",
[task_uuid])
except Exception as ex:
log.exception()
return error("Error while tasking %d@%s: %s" % (participant, part_name, ex))
# Enable the task so the scheduler will schedule it.
try:
dbcursor_query("SELECT api_task_enable(%s)", [task_uuid])
except Exception as ex:
log.exception()
return error("Failed to enable task %s. See system logs." % task_uuid)
log.debug("Task enabled for scheduling.")
return ok_json("%s/%s" % (request.base_url, task_uuid))
else:
return not_allowed()
def tasks():
if request.method == 'GET':
where_clause = "TRUE"
args = []
try:
json_query = arg_json("json")
except ValueError as ex:
return bad_request(str(ex))
if json_query is not None:
where_clause += " AND task.json @> %s"
args.append(request.args.get("json"))
where_clause += " ORDER BY added"
try:
tasks = __tasks_get_filtered(request.base_url,
where_clause=where_clause,
args=args,
expanded=is_expanded(),
detail=arg_boolean("detail"),
single=False)
except Exception as ex:
return error(str(ex))
return ok_json(tasks)
elif request.method == 'POST':
try:
task = pscheduler.json_load(request.data, max_schema=1)
except ValueError as ex:
return bad_request("Invalid task specification: %s" % (str(ex)))
# Validate the JSON against a TaskSpecification
# TODO: Figure out how to do this without the intermediate object
valid, message = pscheduler.json_validate({"": task}, {
"type": "object",
"properties": {
"": {
"$ref": "#/pScheduler/TaskSpecification"
}
},
"required": [""]
})
if not valid:
return bad_request("Invalid task specification: %s" % (message))
# See if the test spec is valid
try:
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test",
task['test']['type'], "spec-is-valid"
],
stdin=pscheduler.json_dump(task['test']['spec']))
if returncode != 0:
return error("Unable to validate test spec: %s" % (stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
if not validate_json["valid"]:
return bad_request(
"Invalid test specification: %s" %
(validate_json.get("error", "Unspecified error")))
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
log.debug("Validated test: %s", pscheduler.json_dump(task['test']))
# Reject tasks that have archive specs that use transforms.
# See ticket #330.
try:
for archive in task['archives']:
if "transform" in archive:
return bad_request(
"Use of transforms in archives is not yet supported.")
except KeyError:
pass # Not there
# Find the participants
try:
# HACK: BWCTLBC
if "lead-bind" in task:
lead_bind_env = {
"PSCHEDULER_LEAD_BIND_HACK": task["lead-bind"]
}
else:
lead_bind_env = None
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test",
task['test']['type'], "participants"
],
stdin=pscheduler.json_dump(task['test']['spec']),
timeout=5,
env_add=lead_bind_env)
if returncode != 0:
return error("Unable to determine participants: " + stderr)
participants = [
host if host is not None else server_netloc() for host in
pscheduler.json_load(stdout, max_schema=1)["participants"]
]
except Exception as ex:
return error("Exception while determining participants: " +
str(ex))
nparticipants = len(participants)
# TODO: The participants must be unique. This should be
# verified by fetching the host name from each one.
#
# TOOL SELECTION
#
lead_bind = task.get("lead-bind", None)
# TODO: Need to provide for tool being specified by the task
# package.
tools = []
tool_params = {"test": pscheduler.json_dump(task["test"])}
# HACK: BWCTLBC
if lead_bind is not None:
log.debug("Using lead bind of %s" % str(lead_bind))
tool_params["lead-bind"] = lead_bind
for participant_no in range(0, len(participants)):
participant = participants[participant_no]
try:
# Make sure the other participants are running pScheduler
participant_api = pscheduler.api_url_hostport(participant)
log.debug("Pinging %s" % (participant))
status, result = pscheduler.url_get(participant_api,
throw=False,
timeout=10,
bind=lead_bind)
if status == 400:
raise TaskPostingException(result)
elif status in [ 202, 204, 205, 206, 207, 208, 226,
300, 301, 302, 303, 304, 205, 306, 307, 308 ] \
or ( (status >= 400) and (status <=499) ):
raise TaskPostingException(
"Host is not running pScheduler")
elif status != 200:
raise TaskPostingException("returned status %d: %s" %
(status, result))
# TODO: This will fail with a very large test spec.
status, result = pscheduler.url_get("%s/tools" %
(participant_api),
params=tool_params,
throw=False,
bind=lead_bind)
if status != 200:
raise TaskPostingException("%d: %s" % (status, result))
tools.append(result)
except TaskPostingException as ex:
return error("Error getting tools from %s: %s" \
% (participant, str(ex)))
log.debug("Participant %s offers tools %s", participant, result)
if len(tools) != nparticipants:
return error("Didn't get a full set of tool responses")
if "tools" in task:
tool = pick_tool(tools, pick_from=task['tools'])
else:
tool = pick_tool(tools)
if tool is None:
# TODO: This could stand some additional diagnostics.
return no_can_do(
"Couldn't find a tool in common among the participants.")
task['tool'] = tool
#
# TASK CREATION
#
tasks_posted = []
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on %s", task["test"])
(processor, whynot) = limitprocessor()
if processor is None:
log.debug("Limit processor is not initialized. %s", whynot)
return no_can_do("Limit processor is not initialized: %s" % whynot)
hints = request_hints()
hints_data = pscheduler.json_dump(hints)
log.debug("Processor = %s" % processor)
passed, limits_passed, diags = processor.process(task["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
# Post the lead with the local database, which also assigns
# its UUID. Make it disabled so the scheduler doesn't try to
# do anything with it until the task has been submitted to all
# of the other participants.
try:
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s, 0, NULL, FALSE)",
[
pscheduler.json_dump(task), participants, hints_data,
pscheduler.json_dump(limits_passed)
],
onerow=True)
except Exception as ex:
return error(str(ex.diag.message_primary))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
task_uuid = cursor.fetchone()[0]
log.debug("Tasked lead, UUID %s", task_uuid)
# Other participants get the UUID and participant list forced upon them.
task["participants"] = participants
task_data = pscheduler.json_dump(task)
for participant in range(1, nparticipants):
part_name = participants[participant]
log.debug("Tasking participant %s", part_name)
try:
# Post the task
log.debug("Tasking %d@%s: %s", participant, part_name,
task_data)
post_url = pscheduler.api_url_hostport(part_name,
'tasks/' + task_uuid)
log.debug("Posting task to %s", post_url)
status, result = pscheduler.url_post(
post_url,
params={'participant': participant},
data=task_data,
bind=lead_bind,
json=False,
throw=False)
log.debug("Remote returned %d: %s", status, result)
if status != 200:
raise TaskPostingException(
"Unable to post task to %s: %s" % (part_name, result))
tasks_posted.append(result)
# Fetch the task's details and add the list of limits
# passed to our own.
status, result = pscheduler.url_get(post_url,
params={"detail": True},
bind=lead_bind,
throw=False)
if status != 200:
raise TaskPostingException(
"Unable to fetch posted task from %s: %s" %
(part_name, result))
log.debug("Fetched %s", result)
try:
details = result["detail"]["spec-limits-passed"]
log.debug("Details from %s: %s", post_url, details)
limits_passed.extend(details)
except KeyError:
pass
except TaskPostingException as ex:
# Disable the task locally and let it get rid of the
# other participants.
posted_to = "%s/%s" % (request.url, task_uuid)
parsed = list(urlparse.urlsplit(posted_to))
parsed[1] = "%s"
template = urlparse.urlunsplit(parsed)
try:
dbcursor_query("SELECT api_task_disable(%s, %s)",
[task_uuid, template])
except Exception:
log.exception()
return error("Error while tasking %s: %s" % (part_name, ex))
# Update the list of limits passed in the local database
# TODO: How do the other participants know about this?
log.debug("Limits passed: %s", limits_passed)
try:
cursor = dbcursor_query(
"UPDATE task SET limits_passed = %s::JSON WHERE uuid = %s",
[pscheduler.json_dump(limits_passed), task_uuid])
except Exception as ex:
return error(str(ex.diag.message_primary))
# Enable the task so the scheduler will schedule it.
try:
dbcursor_query("SELECT api_task_enable(%s)", [task_uuid])
except Exception as ex:
log.exception()
return error("Failed to enable task %s. See system logs." %
task_uuid)
log.debug("Task enabled for scheduling.")
return ok_json("%s/%s" % (request.base_url, task_uuid))
else:
return not_allowed()
def tasks():
if request.method == 'GET':
where_clause = "TRUE"
args = []
try:
json_query = arg_json("json")
except ValueError as ex:
return bad_request(str(ex))
if json_query is not None:
where_clause += " AND task.json_detail @> %s"
args.append(request.args.get("json"))
where_clause += " ORDER BY added"
tasks = __tasks_get_filtered(request.base_url,
where_clause=where_clause,
args=args,
expanded=is_expanded(),
detail=arg_boolean("detail"),
single=False)
return ok_json(tasks)
elif request.method == 'POST':
try:
task = pscheduler.json_load(request.data, max_schema=3)
except ValueError as ex:
return bad_request("Invalid task specification: %s" % (str(ex)))
# Validate the JSON against a TaskSpecification
# TODO: Figure out how to do this without the intermediate object
valid, message = pscheduler.json_validate({"": task}, {
"type": "object",
"properties": {
"": {
"$ref": "#/pScheduler/TaskSpecification"
}
},
"required": [""]
})
if not valid:
return bad_request("Invalid task specification: %s" % (message))
# See if the test spec is valid
try:
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test",
task['test']['type'], "spec-is-valid"
],
stdin=pscheduler.json_dump(task['test']['spec']))
if returncode != 0:
return error("Unable to validate test spec: %s" % (stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
if not validate_json["valid"]:
return bad_request(
"Invalid test specification: %s" %
(validate_json.get("error", "Unspecified error")))
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
log.debug("Validated test: %s", pscheduler.json_dump(task['test']))
# Validate the archives
for archive in task.get("archives", []):
# Data
try:
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "archiver",
archive["archiver"], "data-is-valid"
],
stdin=pscheduler.json_dump(archive["data"]),
)
if returncode != 0:
return error("Unable to validate archive spec: %s" %
(stderr))
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
try:
returned_json = pscheduler.json_load(stdout)
if not returned_json["valid"]:
return bad_request("Invalid archiver data: %s" %
(returned_json["error"]))
except Exception as ex:
return error("Internal probelm validating archiver data: %s" %
(str(ex)))
# Transform, if there was one.
if "transform" in archive:
transform = archive["transform"]
try:
_ = pscheduler.JQFilter(filter_spec=transform["script"],
args=transform.get("args", {}))
except ValueError as ex:
return error("Invalid transform: %s" % (str(ex)))
# Validate the lead binding if there was one.
lead_bind = task.get("lead-bind", None)
if lead_bind is not None \
and (pscheduler.address_interface(lead_bind) is None):
return bad_request("Lead bind '%s' is not on this host" %
(lead_bind))
# Evaluate the task against the limits and reject the request
# if it doesn't pass. We do this early so anything else in
# the process gets any rewrites.
log.debug("Checking limits on %s", task)
(processor, whynot) = limitprocessor()
if processor is None:
log.debug("Limit processor is not initialized. %s", whynot)
return no_can_do("Limit processor is not initialized: %s" % whynot)
hints, error_response = request_hints()
if hints is None:
log.debug("Can't come up with valid hints for lead task limits.")
return error_response
hints_data = pscheduler.json_dump(hints)
log.debug("Processor = %s" % processor)
passed, limits_passed, diags, new_task, _priority \
= processor.process(task, hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
if new_task is not None:
try:
task = new_task
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test",
task['test']['type'], "spec-is-valid"
],
stdin=pscheduler.json_dump(task["test"]["spec"]))
if returncode != 0:
return error(
"Failed to validate rewritten test specification: %s" %
(stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
if not validate_json["valid"]:
return bad_request(
"Rewritten test specification is invalid: %s" %
(validate_json.get("error", "Unspecified error")))
except Exception as ex:
return error(
"Unable to validate rewritten test specification: " +
str(ex))
# Find the participants
try:
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test",
task['test']['type'], "participants"
],
stdin=pscheduler.json_dump(task['test']['spec']),
timeout=5)
if returncode != 0:
return error("Unable to determine participants: " + stderr)
participants = [
host if host is not None else server_netloc() for host in
pscheduler.json_load(stdout, max_schema=1)["participants"]
]
except Exception as ex:
return error("Exception while determining participants: " +
str(ex))
nparticipants = len(participants)
# TODO: The participants must be unique. This should be
# verified by fetching the host name from each one.
#
# TOOL SELECTION
#
# TODO: Need to provide for tool being specified by the task
# package.
tools = []
tool_params = {"test": pscheduler.json_dump(task["test"])}
tool_offers = {}
for participant_no in range(0, len(participants)):
participant = participants[participant_no]
try:
# Make sure the other participants are running pScheduler
participant_api = pscheduler.api_url_hostport(participant)
log.debug("Pinging %s" % (participant))
status, result = pscheduler.url_get(participant_api,
throw=False,
timeout=10,
bind=lead_bind)
if status == 400:
raise TaskPostingException(result)
elif status in [ 202, 204, 205, 206, 207, 208, 226,
300, 301, 302, 303, 304, 205, 306, 307, 308 ] \
or ( (status >= 400) and (status <=499) ):
raise TaskPostingException(
"Host is not running pScheduler")
elif status != 200:
raise TaskPostingException("returned status %d: %s" %
(status, result))
# TODO: This will fail with a very large test spec.
status, result = pscheduler.url_get("%s/tools" %
(participant_api),
params=tool_params,
throw=False,
bind=lead_bind)
if status != 200:
raise TaskPostingException("%d: %s" % (status, result))
tools.append(result)
except TaskPostingException as ex:
return error("Error getting tools from %s: %s" \
% (participant, str(ex)))
log.debug("Participant %s offers tools %s", participant, result)
tool_offers[participant] = result
if len(tools) != nparticipants:
return error("Didn't get a full set of tool responses")
if "tools" in task:
tool = pick_tool(tools, pick_from=task['tools'])
else:
tool = pick_tool(tools)
# Complain if no usable tool was found
if tool is None:
offers = []
for participant in participants:
participant_offers = tool_offers.get(participant,
[{
"name": "nothing"
}])
if participant_offers is not None:
offer_set = [offer["name"] for offer in participant_offers]
else:
offer_set = ["nothing"]
offers.append("%s offered %s" %
(participant, ", ".join(offer_set)))
return no_can_do("No tool in common among the participants: %s." %
("; ".join(offers)))
task['tool'] = tool
#
# TASK CREATION
#
tasks_posted = []
# Post the lead with the local database, which also assigns
# its UUID. Make it disabled so the scheduler doesn't try to
# do anything with it until the task has been submitted to all
# of the other participants.
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s, 0, %s, NULL, FALSE, %s)",
[
pscheduler.json_dump(task), participants, hints_data,
pscheduler.json_dump(limits_passed),
task.get("priority", None), diags
],
onerow=True)
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
task_uuid = cursor.fetchone()[0]
log.debug("Tasked lead, UUID %s", task_uuid)
# Other participants get the UUID and participant list forced upon them.
task["participants"] = participants
task_params = {"key": task["_key"]} if "_key" in task else {}
for participant in range(1, nparticipants):
part_name = participants[participant]
log.debug("Tasking participant %s", part_name)
try:
# Post the task
log.debug("Tasking %d@%s: %s", participant, part_name, task)
post_url = pscheduler.api_url_hostport(part_name,
'tasks/' + task_uuid)
task_params["participant"] = participant
log.debug("Posting task to %s", post_url)
status, result = pscheduler.url_post(post_url,
params=task_params,
data=task,
bind=lead_bind,
json=False,
throw=False)
log.debug("Remote returned %d: %s", status, result)
if status != 200:
raise TaskPostingException(
"Unable to post task to %s: %s" % (part_name, result))
tasks_posted.append(result)
# Fetch the task's details and add the list of limits
# passed to our own.
status, result = pscheduler.url_get(post_url,
params={"detail": True},
bind=lead_bind,
throw=False)
if status != 200:
raise TaskPostingException(
"Unable to fetch posted task from %s: %s" %
(part_name, result))
log.debug("Fetched %s", result)
try:
details = result["detail"]["spec-limits-passed"]
log.debug("Details from %s: %s", post_url, details)
limits_passed.extend(details)
except KeyError:
pass
except TaskPostingException as ex:
# Disable the task locally and let it get rid of the
# other participants.
posted_to = "%s/%s" % (request.url, task_uuid)
parsed = list(urlparse.urlsplit(posted_to))
parsed[1] = "%s"
template = urlparse.urlunsplit(parsed)
try:
dbcursor_query("SELECT api_task_disable(%s, %s)",
[task_uuid, template])
except Exception:
log.exception()
return error("Error while tasking %s: %s" % (part_name, ex))
# Update the list of limits passed in the local database
# TODO: How do the other participants know about this?
log.debug("Limits passed: %s", limits_passed)
cursor = dbcursor_query(
"UPDATE task SET limits_passed = %s::JSON WHERE uuid = %s",
[pscheduler.json_dump(limits_passed), task_uuid])
# Enable the task so the scheduler will schedule it.
try:
dbcursor_query("SELECT api_task_enable(%s)", [task_uuid])
except Exception as ex:
log.exception()
return error("Failed to enable task %s. See system logs." %
task_uuid)
log.debug("Task enabled for scheduling.")
task_url = "%s/%s" % (request.base_url, task_uuid)
# Non-expanded gets just the URL
if not arg_boolean("expanded"):
return ok_json(task_url)
# Expanded gets a redirect to GET+expanded
params = []
for arg in ["detail", "pretty"]:
if arg_boolean(arg):
params.append(arg)
if params:
task_url += "?%s" % ("&".join(params))
return see_other(task_url)
else:
return not_allowed()
def clock_state():
"""
Determine the state of the system clock and return a hash of
information conforming to the definition of a SystemClockStatus
object as described in the JSON dictionary.
time - Current system time as an ISO 8601 string
synchronized - Whether or not the clock is synchronized to an
outside source.
source - The source of synchronization. Currently, the only valid
value is "ntp." Not provided if not synchronized.
reference - A human-readable string describing the source. Not
provided if not synchronized.
offset - A float indicating the estimated clock offset. Not
provided if not synchronized.
error -
"""
status, stdout, stderr = pscheduler.run_program(["chronyc", " tracking"])
if "FileNotFound" not in stderr:
if "506 Cannot talk to daemon" not in stdout:
return chrony_clock_state(stdout, stderr)
adjtime = ntp_adjtime()
system_synchronized = adjtime.synchronized
# Format the local time with offset as ISO 8601. Python's
# strftime() only does "-0400" format; we need "-04:00".
utc = datetime.datetime.utcnow()
local_tz = tzlocal.get_localzone()
time_here = pytz.utc.localize(utc).astimezone(local_tz)
raw_offset = time_here.strftime("%z")
if len(raw_offset):
offset = raw_offset[:3] + ":" + raw_offset[-2:]
else:
offset = ""
result = {
"time": time_here.strftime("%Y-%m-%dT%H:%M:%S.%f") + offset,
"synchronized": system_synchronized
}
if system_synchronized:
# Assume NTP for the time being
try:
ntp = ntplib.NTPClient().request("127.0.0.1")
result["offset"] = ntp.offset
result["source"] = "ntp"
result["reference"] = "%s from %s" % (ntplib.stratum_to_text(
ntp.stratum), ntplib.ref_id_to_text(ntp.ref_id))
except Exception as ex:
result["synchronized"] = False
result["error"] = str(ex)
return result
def tasks_uuid_runs_run_result(task, run):
if task is None:
return bad_request("Missing or invalid task")
if run is None:
return bad_request("Missing or invalid run")
wait = arg_boolean('wait')
format = request.args.get('format')
if format is None:
format = 'application/json'
if format not in [ 'application/json', 'text/html', 'text/plain' ]:
return bad_request("Unsupported format " + format)
# If asked for 'first', dig up the first run and use its UUID.
# This is more for debug convenience than anything else.
if run == 'first':
try:
run = __runs_first_run(task)
except Exception as ex:
log.exception()
return error(str(ex))
if run is None:
return not_found()
#
# Camp on the run for a result
#
# 40 tries at 0.25s intervals == 10 sec.
tries = 40 if wait else 1
while tries:
try:
cursor = dbcursor_query("""
SELECT
test.name,
run.result_merged,
task.json #> '{test, spec}'
FROM
run
JOIN task ON task.id = run.task
JOIN test ON test.id = task.test
WHERE
task.uuid = %s
AND run.uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
# TODO: Make sure we got back one row with two columns.
row = cursor.fetchone()
if not wait and row[1] is None:
time.sleep(0.25)
tries -= 1
else:
break
if tries == 0:
return not_found()
test_type, merged_result, test_spec = row
# JSON requires no formatting.
if format == 'application/json':
return ok_json(merged_result)
if not merged_result['succeeded']:
if format == 'text/plain':
return ok("Test failed.", mimetype=format)
elif format == 'text/html':
return ok("<p>Test failed.</p>", mimetype=format)
return error("Unsupported format " + format)
formatter_input = {
"spec": test_spec,
"result": merged_result
}
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", test_type,
"result-format", format ],
stdin = pscheduler.json_dump(formatter_input)
)
if returncode != 0:
return error("Failed to format result: " + stderr)
return ok(stdout.rstrip(), mimetype=format)
