def create_metadata(self, metadata):
result = {}
post_url = self.url
if not post_url.endswith('/'):
post_url += '/'
log.debug("Posting metadata to %s: %s" % (post_url, metadata))
status_code, result = pscheduler.url_post(
post_url,
data=pscheduler.json_dump(metadata),
headers=self.headers,
throw=False,
json=True,
verify_keys=self.verify_ssl,
bind=self.bind,
allow_redirects=False,
timeout=HTTP_TIMEOUT)
log.debug("Esmond returned HTTP {0}: {1}".format(status_code, result))
if status_code == 301:
#redirects
return False, "Server is attempting to redirect archive URL %s. If redirecting to https, please change archive URL to https instead of relying on redirection." % post_url
elif status_code not in [200, 201]:
try:
result_json = pscheduler.json_load(result)
except:
result_json = {"detail": "Invalid JSON returned"}
return False, "%d: %s" % (
status_code,
result_json.get("detail", pscheduler.json_dump(result_json)))
return True, result
def handle_run_error(msg, diags=None, do_log=True):
if do_log:
log.error(msg)
results = {
'schema': LATENCY_SCHEMA_VERSION,
'succeeded': False,
'error': msg,
'diags': diags
}
print pscheduler.json_dump(results)
print pscheduler.api_result_delimiter()
sys.stdout.flush()
def handle_run_error(msg, diags=None, do_log=True):
if do_log:
log.error(msg)
results = {
'schema': LATENCY_SCHEMA_VERSION,
'succeeded': False,
'error': msg,
'diags': diags
}
print pscheduler.json_dump(results)
print pscheduler.api_result_delimiter()
sys.stdout.flush()
def create_data(self, metadata_key, data_points):
result = {}
put_url = self.url
if not put_url.endswith('/'):
put_url += '/'
put_url += ("%s/" % metadata_key)
data = {'data': data_points}
log.debug("Putting data to %s: %s" % (put_url, data))
status_code, result = pscheduler.url_put(put_url,
data=data,
headers=self.headers,
json=True,
throw=False,
verify_keys=self.verify_ssl,
bind=self.bind,
allow_redirects=False,
timeout=HTTP_TIMEOUT)
if status_code == 409:
#duplicate data
log.debug("Attempted to add duplicate data point. Skipping")
elif status_code == 301:
#redirects
return False, "Server is attempting to redirect archive URL %s. If redirecting to https, please change archive URL to https instead of relying on redirection." % put_url
elif status_code not in [200, 201]:
try:
result_json = pscheduler.json_load(result)
except:
result_json = {"detail": "Invalid JSON returned"}
return False, "%d: %s" % (
status_code,
result_json.get("detail", pscheduler.json_dump(result_json)))
return True, ""
def evaluate(
self,
run # The proposed run
):
# Dissent if the test isn't our type
if run["type"] != self.test:
return {
"passed": False,
"reasons": ["Test is not '%s'" % self.test]
}
pass_input = {"spec": run["spec"], "limit": self.limit}
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test", self.test,
"limit-passes"
],
stdin=pscheduler.json_dump(pass_input),
# TODO: Is this reasonable?
timeout=5)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
check_result = pscheduler.json_load(stdout, max_schema=1)
passed = check_result["passes"]
result = {"passed": passed, "limit": self.limit}
if not passed:
result["reasons"] = check_result["errors"]
return result
def tests_name_spec(name):
try:
cursor = dbcursor_query("SELECT EXISTS (SELECT * FROM test WHERE NAME = %s)",
[ name ])
except Exception as ex:
return error(str(ex))
exists = cursor.fetchone()[0]
if not exists:
return not_found()
try:
args = arg_json('args')
except ValueError:
return error("Invalid JSON passed to 'args'")
status, stdout, stderr = pscheduler.run_program(
[ 'pscheduler', 'internal', 'invoke', 'test', name, 'cli-to-spec' ],
stdin = pscheduler.json_dump(args),
short = True,
)
if status != 0:
return bad_request(stderr)
# The extra parse here makes 'pretty' work.
returned_json = pscheduler.json_load(stdout)
return ok_json(returned_json)
def tests_name_spec(name):
cursor = dbcursor_query("SELECT EXISTS (SELECT * FROM test"
" WHERE available AND name = %s)",
[ name ])
exists = cursor.fetchone()[0]
cursor.close()
if not exists:
return not_found()
try:
args = arg_json('args')
except ValueError as ex:
return bad_request("JSON passed to 'args': %s " % (str(ex)))
status, stdout, stderr = pscheduler.run_program(
[ 'pscheduler', 'internal', 'invoke', 'test', name, 'cli-to-spec' ],
stdin = pscheduler.json_dump(args),
timeout=5
)
if status != 0:
return bad_request(stderr)
# The extra parse here makes 'pretty' work.
returned_json = pscheduler.json_load(stdout)
return ok_json(returned_json, sanitize=False)
def __init__(
self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.test = data['test']
self.limit = data['limit']
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test", self.test,
"limit-is-valid"
],
stdin=pscheduler.json_dump(self.limit),
# TODO: Is this reasonable?
timeout=5)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
result = pscheduler.json_load(stdout, max_schema=1)
if not result['valid']:
raise ValueError("Invalid limit: %s" % result['message'])
def evaluate(self, run): # The proposed run
# Dissent if the test isn't our type
if run["type"] != self.test:
return {"passed": False, "reasons": ["Test is not '%s'" % self.test]}
pass_input = {"spec": run["spec"], "limit": self.limit}
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", self.test, "limit-passes"],
stdin=pscheduler.json_dump(pass_input),
# TODO: Is this reasonable?
timeout=5,
)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
check_result = pscheduler.json_load(stdout)
passed = check_result["passes"]
result = {"passed": passed}
if not passed:
result["reasons"] = check_result["errors"]
return result
def create_data(self, metadata_key, data_points):
result = {}
put_url = self.url
if not put_url.endswith('/'):
put_url += '/'
put_url += ("%s/" % metadata_key)
data = { 'data': data_points }
log.debug("Putting data to %s: %s" % (put_url, data))
r = requests.put(put_url, data=pscheduler.json_dump(data), headers=self.headers, verify=self.verify_ssl)
if r.status_code== 409:
#duplicate data
log.debug("Attempted to add duplicate data point. Skipping")
elif r.status_code != 200 and r.status_code != 201:
try:
return False, "%d: %s" % (r.status_code, pscheduler.json_load(r.text)['detail'])
except:
return False, "%d: %s" % (r.status_code, r.text)
return True, ""
def create_metadata(self, metadata):
result = {}
post_url = self.url
if not post_url.endswith('/'):
post_url += '/'
log.debug("Posting metadata to %s: %s" % (post_url, metadata))
r = requests.post(post_url, data=pscheduler.json_dump(metadata), headers=self.headers, verify=self.verify_ssl)
if r.status_code != 200 and r.status_code != 201:
try:
return False, "%d: %s" % (r.status_code, pscheduler.json_load(r.text)['detail'])
except:
return False, "%d: %s" % (r.status_code, r.text)
try:
rjson = pscheduler.json_load(r.text)
log.debug("Metadata POST result: %s" % rjson)
except:
return False, "Invalid JSON returned from server: %s" % r.text
return True, rjson
def tasks_uuid_cli(uuid):
if not uuid_is_valid(uuid):
return not_found()
# Get a task, adding server-derived details if a 'detail'
# argument is present.
try:
cursor = dbcursor_query(
"""SELECT
task.json #>> '{test, spec}',
test.name
FROM
task
JOIN test on test.id = task.test
WHERE task.uuid = %s""", [uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if row is None:
return not_found()
json, test = row
try:
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", test, "spec-to-cli"],
stdin=json)
if returncode != 0:
return error("Unable to convert test spec: " + stderr)
except Exception as ex:
return error("Unable to convert test spec: " + str(ex))
returned = pscheduler.json_load(stdout)
returned.insert(0, test)
return ok(pscheduler.json_dump(returned))
def __call__(self, task, classifiers, validate_callback=None):
"""
Rewrite the task given the classifiers. Returns a tuple
containing the rewritten task and an array of diagnostic
messages.
Returns a tuple containing a boolean indicating whether or not
the task was changed, the revised task and an array of strings
containing diagnostic information.
The caller is expected to catch and deal with any
JQRuntimeError that is thrown.
"""
task_in = copy.deepcopy(task)
# Rewriter-private data
task_in[self.PRIVATE_KEY] = {
"classifiers": classifiers,
"changed": False,
"diags": []
}
result = self.transform(task_in)[0]
# print "RES", pscheduler.json_dump(result, pretty=True)
if ("test" not in result) \
or ("type" not in result["test"]) \
or (not isinstance(result["test"]["type"], basestring)) \
or ("spec" not in result["test"]) \
or (result["test"]["type"] != task["test"]["type"]):
raise ValueError("Invalid rewriter result:\n%s" \
% pscheduler.json_dump(result))
changed = result[self.PRIVATE_KEY]["changed"]
diags = result[self.PRIVATE_KEY]["diags"]
del result[self.PRIVATE_KEY]
return changed, result, diags
def __init__(self, data): # Data suitable for this class
valid, message = test_data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.test = data["test"]
self.limit = data["limit"]
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", self.test, "limit-is-valid"],
stdin=pscheduler.json_dump(self.limit),
# TODO: Is this reasonable?
timeout=5,
)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
result = pscheduler.json_load(stdout)
if not result["valid"]:
raise ValueError("Invalid limit: %s" % result["message"])
def tasks_uuid_cli(uuid):
# Get a task, adding server-derived details if a 'detail'
# argument is present.
try:
cursor = dbcursor_query(
"""SELECT
task.json #>> '{test, spec}',
test.name
FROM
task
JOIN test on test.id = task.test
WHERE task.uuid = %s""", [uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if row is None:
return not_found()
json, test = row
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
test, "spec-to-cli" ], stdin = json )
if returncode != 0:
return error("Unable to convert test spec: " + stderr)
except Exception as ex:
return error("Unable to convert test spec: " + str(ex))
returned = pscheduler.json_load(stdout)
returned.insert(0, test)
return ok(pscheduler.json_dump(returned))
def json_dump(dump):
return pscheduler.json_dump(dump, pretty=arg_boolean("pretty"))
def tasks_uuid(uuid):
if not uuid_is_valid(uuid):
return not_found()
if request.method == 'GET':
try:
tasks = __tasks_get_filtered(request.base_url,
where_clause="task.uuid = %s",
args=[uuid],
expanded=True,
detail=arg_boolean("detail"),
single=True)
except Exception as ex:
return error(str(ex))
if not tasks:
return not_found()
return ok_json(tasks[0])
elif request.method == 'POST':
log.debug("Posting to %s", uuid)
log.debug("Data is %s", request.data)
# TODO: This is only for participant 1+
# TODO: This should probably a PUT and not a POST.
try:
json_in = pscheduler.json_load(request.data, max_schema=1)
except ValueError:
return bad_request("Invalid JSON")
log.debug("JSON is %s", json_in)
try:
participant = arg_cardinal('participant')
except ValueError as ex:
return bad_request("Invalid participant: " + str(ex))
log.debug("Participant %d", participant)
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on task")
processor, whynot = limitprocessor()
if processor is None:
message = "Limit processor is not initialized: %s" % whynot
log.debug(message)
return no_can_do(message)
hints = request_hints()
hints_data = pscheduler.json_dump(hints)
passed, limits_passed, diags = processor.process(
json_in["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
log.debug("Limits passed")
# TODO: Pluck UUID from URI
uuid = url_last_in_path(request.url)
log.debug("Posting task %s", uuid)
try:
try:
participants = pscheduler.json_load(
request.data, max_schema=1)["participants"]
except:
return bad_request("No participants provided")
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s, %s, %s, TRUE)", [
request.data, participants, hints_data,
pscheduler.json_dump(limits_passed), participant, uuid
])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
# TODO: Assert that rowcount is 1
log.debug("All done: %s", base_url())
return ok(base_url())
elif request.method == 'DELETE':
parsed = list(urlparse.urlsplit(request.url))
parsed[1] = "%s"
template = urlparse.urlunsplit(parsed)
try:
requester = task_requester(uuid)
if requester is None:
return not_found()
if not access_write_ok(requester):
return forbidden()
cursor = dbcursor_query("SELECT api_task_disable(%s, %s)",
[uuid, template])
cursor.close()
except Exception as ex:
return error(str(ex))
return ok()
else:
return not_allowed()
def tasks():
if request.method == 'GET':
where_clause = "TRUE"
args = []
try:
json_query = arg_json("json")
except ValueError as ex:
return bad_request(str(ex))
if json_query is not None:
where_clause += " AND task.json @> %s"
args.append(request.args.get("json"))
where_clause += " ORDER BY added"
try:
tasks = __tasks_get_filtered(request.base_url,
where_clause=where_clause,
args=args,
expanded=is_expanded(),
detail=arg_boolean("detail"),
single=False)
except Exception as ex:
return error(str(ex))
return ok_json(tasks)
elif request.method == 'POST':
try:
task = pscheduler.json_load(request.data, max_schema=1)
except ValueError as ex:
return bad_request("Invalid task specification: %s" % (str(ex)))
# Validate the JSON against a TaskSpecification
# TODO: Figure out how to do this without the intermediate object
valid, message = pscheduler.json_validate({"": task}, {
"type": "object",
"properties": {
"": {
"$ref": "#/pScheduler/TaskSpecification"
}
},
"required": [""]
})
if not valid:
return bad_request("Invalid task specification: %s" % (message))
# See if the test spec is valid
try:
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test",
task['test']['type'], "spec-is-valid"
],
stdin=pscheduler.json_dump(task['test']['spec']))
if returncode != 0:
return error("Unable to validate test spec: %s" % (stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
if not validate_json["valid"]:
return bad_request(
"Invalid test specification: %s" %
(validate_json.get("error", "Unspecified error")))
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
log.debug("Validated test: %s", pscheduler.json_dump(task['test']))
# Reject tasks that have archive specs that use transforms.
# See ticket #330.
try:
for archive in task['archives']:
if "transform" in archive:
return bad_request(
"Use of transforms in archives is not yet supported.")
except KeyError:
pass # Not there
# Find the participants
try:
# HACK: BWCTLBC
if "lead-bind" in task:
lead_bind_env = {
"PSCHEDULER_LEAD_BIND_HACK": task["lead-bind"]
}
else:
lead_bind_env = None
returncode, stdout, stderr = pscheduler.run_program(
[
"pscheduler", "internal", "invoke", "test",
task['test']['type'], "participants"
],
stdin=pscheduler.json_dump(task['test']['spec']),
timeout=5,
env_add=lead_bind_env)
if returncode != 0:
return error("Unable to determine participants: " + stderr)
participants = [
host if host is not None else server_netloc() for host in
pscheduler.json_load(stdout, max_schema=1)["participants"]
]
except Exception as ex:
return error("Exception while determining participants: " +
str(ex))
nparticipants = len(participants)
# TODO: The participants must be unique. This should be
# verified by fetching the host name from each one.
#
# TOOL SELECTION
#
lead_bind = task.get("lead-bind", None)
# TODO: Need to provide for tool being specified by the task
# package.
tools = []
tool_params = {"test": pscheduler.json_dump(task["test"])}
# HACK: BWCTLBC
if lead_bind is not None:
log.debug("Using lead bind of %s" % str(lead_bind))
tool_params["lead-bind"] = lead_bind
for participant_no in range(0, len(participants)):
participant = participants[participant_no]
try:
# Make sure the other participants are running pScheduler
participant_api = pscheduler.api_url_hostport(participant)
log.debug("Pinging %s" % (participant))
status, result = pscheduler.url_get(participant_api,
throw=False,
timeout=10,
bind=lead_bind)
if status == 400:
raise TaskPostingException(result)
elif status in [ 202, 204, 205, 206, 207, 208, 226,
300, 301, 302, 303, 304, 205, 306, 307, 308 ] \
or ( (status >= 400) and (status <=499) ):
raise TaskPostingException(
"Host is not running pScheduler")
elif status != 200:
raise TaskPostingException("returned status %d: %s" %
(status, result))
# TODO: This will fail with a very large test spec.
status, result = pscheduler.url_get("%s/tools" %
(participant_api),
params=tool_params,
throw=False,
bind=lead_bind)
if status != 200:
raise TaskPostingException("%d: %s" % (status, result))
tools.append(result)
except TaskPostingException as ex:
return error("Error getting tools from %s: %s" \
% (participant, str(ex)))
log.debug("Participant %s offers tools %s", participant, result)
if len(tools) != nparticipants:
return error("Didn't get a full set of tool responses")
if "tools" in task:
tool = pick_tool(tools, pick_from=task['tools'])
else:
tool = pick_tool(tools)
if tool is None:
# TODO: This could stand some additional diagnostics.
return no_can_do(
"Couldn't find a tool in common among the participants.")
task['tool'] = tool
#
# TASK CREATION
#
tasks_posted = []
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on %s", task["test"])
(processor, whynot) = limitprocessor()
if processor is None:
log.debug("Limit processor is not initialized. %s", whynot)
return no_can_do("Limit processor is not initialized: %s" % whynot)
hints = request_hints()
hints_data = pscheduler.json_dump(hints)
log.debug("Processor = %s" % processor)
passed, limits_passed, diags = processor.process(task["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
# Post the lead with the local database, which also assigns
# its UUID. Make it disabled so the scheduler doesn't try to
# do anything with it until the task has been submitted to all
# of the other participants.
try:
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s, 0, NULL, FALSE)",
[
pscheduler.json_dump(task), participants, hints_data,
pscheduler.json_dump(limits_passed)
],
onerow=True)
except Exception as ex:
return error(str(ex.diag.message_primary))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
task_uuid = cursor.fetchone()[0]
log.debug("Tasked lead, UUID %s", task_uuid)
# Other participants get the UUID and participant list forced upon them.
task["participants"] = participants
task_data = pscheduler.json_dump(task)
for participant in range(1, nparticipants):
part_name = participants[participant]
log.debug("Tasking participant %s", part_name)
try:
# Post the task
log.debug("Tasking %d@%s: %s", participant, part_name,
task_data)
post_url = pscheduler.api_url_hostport(part_name,
'tasks/' + task_uuid)
log.debug("Posting task to %s", post_url)
status, result = pscheduler.url_post(
post_url,
params={'participant': participant},
data=task_data,
bind=lead_bind,
json=False,
throw=False)
log.debug("Remote returned %d: %s", status, result)
if status != 200:
raise TaskPostingException(
"Unable to post task to %s: %s" % (part_name, result))
tasks_posted.append(result)
# Fetch the task's details and add the list of limits
# passed to our own.
status, result = pscheduler.url_get(post_url,
params={"detail": True},
bind=lead_bind,
throw=False)
if status != 200:
raise TaskPostingException(
"Unable to fetch posted task from %s: %s" %
(part_name, result))
log.debug("Fetched %s", result)
try:
details = result["detail"]["spec-limits-passed"]
log.debug("Details from %s: %s", post_url, details)
limits_passed.extend(details)
except KeyError:
pass
except TaskPostingException as ex:
# Disable the task locally and let it get rid of the
# other participants.
posted_to = "%s/%s" % (request.url, task_uuid)
parsed = list(urlparse.urlsplit(posted_to))
parsed[1] = "%s"
template = urlparse.urlunsplit(parsed)
try:
dbcursor_query("SELECT api_task_disable(%s, %s)",
[task_uuid, template])
except Exception:
log.exception()
return error("Error while tasking %s: %s" % (part_name, ex))
# Update the list of limits passed in the local database
# TODO: How do the other participants know about this?
log.debug("Limits passed: %s", limits_passed)
try:
cursor = dbcursor_query(
"UPDATE task SET limits_passed = %s::JSON WHERE uuid = %s",
[pscheduler.json_dump(limits_passed), task_uuid])
except Exception as ex:
return error(str(ex.diag.message_primary))
# Enable the task so the scheduler will schedule it.
try:
dbcursor_query("SELECT api_task_enable(%s)", [task_uuid])
except Exception as ex:
log.exception()
return error("Failed to enable task %s. See system logs." %
task_uuid)
log.debug("Task enabled for scheduling.")
return ok_json("%s/%s" % (request.base_url, task_uuid))
else:
return not_allowed()
if len(raw_offset):
offset = raw_offset[:3] + ":" + raw_offset[-2:]
else:
offset = ""
result = {
"time": time_here.strftime("%Y-%m-%dT%H:%M:%S.%f") + offset,
"synchronized": system_synchronized
}
if system_synchronized:
# Assume NTP for the time being
try:
ntp = ntplib.NTPClient().request("127.0.0.1")
result["offset"] = ntp.offset
result["source"] = "ntp"
result["reference"] = "%s from %s" % (ntplib.stratum_to_text(
ntp.stratum), ntplib.ref_id_to_text(ntp.ref_id))
except Exception as ex:
result["synchronized"] = False
result["error"] = str(ex)
return result
if __name__ == "__main__":
import pscheduler
print pscheduler.json_dump(clock_state(), pretty=True)
def response_json_dump(dump):
return pscheduler.json_dump(dump,
pretty=arg_boolean('pretty')
)
def tasks_uuid(uuid):
if not uuid_is_valid(uuid):
return not_found()
if request.method == 'GET':
tasks = __tasks_get_filtered(request.base_url,
where_clause="task.uuid = %s",
args=[uuid],
expanded=True,
detail=arg_boolean("detail"),
single=True)
if not tasks:
return not_found()
return ok_json(tasks[0])
elif request.method == 'POST':
data = request.data.decode("ascii")
log.debug("Posting to %s", uuid)
log.debug("Data is %s", data)
# TODO: This is only for participant 1+
# TODO: This should probably a PUT and not a POST.
try:
json_in = pscheduler.json_load(data, max_schema=3)
except ValueError as ex:
return bad_request("Invalid JSON: %s" % str(ex))
log.debug("JSON is %s", json_in)
try:
participant = arg_cardinal('participant')
except ValueError as ex:
return bad_request("Invalid participant: " + str(ex))
log.debug("Participant %d", participant)
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on task")
processor, whynot = limitprocessor()
if processor is None:
message = "Limit processor is not initialized: %s" % whynot
log.debug(message)
return no_can_do(message)
hints, error_response = request_hints()
if hints is None:
log.debug("Can't come up with valid hints for subordinate limits.")
return error_response
hints_data = pscheduler.json_dump(hints)
# Only the lead rewrites tasks; everyone else just applies
# limits.
passed, limits_passed, diags, _new_task, priority \
= processor.process(json_in, hints, rewrite=False)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
log.debug("Limits passed")
# TODO: Pluck UUID from URI
uuid = url_last_in_path(request.url)
log.debug("Posting task %s", uuid)
try:
try:
participants = pscheduler.json_load(
data, max_schema=3)["participants"]
except Exception as ex:
return bad_request("Task error: %s" % str(ex))
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s, %s, %s, %s, TRUE, %s)",
[
data, participants, hints_data,
pscheduler.json_dump(limits_passed), participant,
json_in.get("priority", None), uuid, "\n".join(diags)
])
except Exception as ex:
return bad_request("Task error: %s" % str(ex))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
# TODO: Assert that rowcount is 1
log.debug("All done: %s", base_url())
return ok(base_url())
elif request.method == 'DELETE':
requester, key = task_requester_key(uuid)
if requester is None:
return not_found()
if not access_write_task(requester, key):
return forbidden()
parsed = list(urllib.parse.urlsplit(request.url))
parsed[1] = "%s"
template = urllib.parse.urlunsplit(parsed)
log.debug("Disabling")
cursor = dbcursor_query("SELECT api_task_disable(%s, %s)",
[uuid, template])
cursor.close()
return ok()
else:
return not_allowed()
def tasks():
if request.method == 'GET':
expanded = is_expanded()
query = """
SELECT json, uuid
FROM task
"""
args = []
try:
json_query = arg_json("json")
except ValueError as ex:
return bad_request(str(ex))
if json_query is not None:
query += "WHERE json @> %s"
args.append(request.args.get("json"))
query += " ORDER BY added"
try:
cursor = dbcursor_query(query, args)
except Exception as ex:
return error(str(ex))
result = []
for row in cursor:
url = base_url(row[1])
if not expanded:
result.append(url)
continue
row[0]['href'] = url
result.append(row[0])
return json_response(result)
elif request.method == 'POST':
try:
task = pscheduler.json_load(request.data)
except ValueError:
return bad_request("Invalid JSON:" + request.data)
# TODO: Validate the JSON against a TaskSpecification
# See if the task spec is valid
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
task['test']['type'], "spec-is-valid" ],
stdin = pscheduler.json_dump(task['test']['spec'])
)
if returncode != 0:
return error("Invalid test specification: " + stderr)
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
log.debug("Validated test: %s", pscheduler.json_dump(task['test']))
# Find the participants
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
task['test']['type'], "participants" ],
stdin = pscheduler.json_dump(task['test']['spec'])
)
if returncode != 0:
return error("Unable to determine participants: " + stderr)
participants = [ host if host is not None
else pscheduler.api_this_host()
for host in pscheduler.json_load(stdout)["participants"] ]
except Exception as ex:
return error("Unable to determine participants: " + str(ex))
nparticipants = len(participants)
# TODO: The participants must be unique. This should be
# verified by fetching the host name from each one.
#
# TOOL SELECTION
#
# TODO: Need to provide for tool being specified by the task
# package.
tools = []
for participant in participants:
try:
# TODO: This will fail with a very large test spec.
status, result = pscheduler.url_get(
pscheduler.api_url(participant, "tools"),
params={ 'test': pscheduler.json_dump(task['test']) }
)
if status != 200:
raise Exception("%d: %s" % (status, result))
tools.append(result)
except Exception as ex:
return error("Error getting tools from %s: %s" \
% (participant, str(ex)))
log.debug("Participant %s offers tools %s", participant, tools)
if len(tools) != nparticipants:
return error("Didn't get a full set of tool responses")
if "tools" in task:
tool = pick_tool(tools, pick_from=task['tools'])
else:
tool = pick_tool(tools)
if tool is None:
# TODO: This could stand some additional diagnostics.
return no_can_do("Couldn't find a tool in common among the participants.")
task['tool'] = tool
#
# TASK CREATION
#
task_data = pscheduler.json_dump(task)
log.debug("Task data: %s", task_data)
tasks_posted = []
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on %s", task["test"])
(processor, whynot) = limitprocessor()
if processor is None:
log.debug("Limit processor is not initialized. %s", whynot)
return no_can_do("Limit processor is not initialized: %s" % whynot)
# TODO: This is cooked up in two places. Make a function of it.
hints = {
"ip": request.remote_addr
}
hints_data = pscheduler.json_dump(hints)
log.debug("Processor = %s" % processor)
passed, diags = processor.process(task["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
# Post the lead with the local database, which also assigns
# its UUID. Make it disabled so the scheduler doesn't try to
# do anything with it until the task has been submitted to all
# of the other participants.
try:
cursor = dbcursor_query("SELECT * FROM api_task_post(%s, %s, 0, NULL, FALSE)",
[task_data, hints_data], onerow=True)
except Exception as ex:
return error(str(ex.diag.message_primary))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
task_uuid = cursor.fetchone()[0]
log.debug("Tasked lead, UUID %s", task_uuid)
# Other participants get the UUID forced upon them.
for participant in range(1,nparticipants):
part_name = participants[participant]
try:
log.debug("Tasking %d@%s: %s", participant, part_name, task_data)
post_url = pscheduler.api_url(part_name,
'tasks/' + task_uuid)
log.debug("Posting task to %s", post_url)
status, result = pscheduler.url_post(
post_url,
params={ 'participant': participant },
data=task_data,
json=False,
throw=False)
log.debug("Remote returned %d: %s", status, result)
if status != 200:
raise Exception("Unable to post task to %s: %s"
% (part_name, result))
tasks_posted.append(result)
except Exception as ex:
log.exception()
for url in tasks_posted:
# TODO: Handle failure?
status, result = requests.delete(url)
try:
dbcursor_query("SELECT api_task_delete(%s)",
[task_uuid])
except Exception as ex:
log.exception()
return error("Error while tasking %d@%s: %s" % (participant, part_name, ex))
# Enable the task so the scheduler will schedule it.
try:
dbcursor_query("SELECT api_task_enable(%s)", [task_uuid])
except Exception as ex:
log.exception()
return error("Failed to enable task %s. See system logs." % task_uuid)
log.debug("Task enabled for scheduling.")
return ok_json("%s/%s" % (request.base_url, task_uuid))
else:
return not_allowed()
def __init__(self, test, nparticipants, a, z, debug=False):
"""
Construct a task runner
"""
self.debug = debug
self.results = {
"hosts": {
"a": a,
"z": z
},
"nparticipants": nparticipants,
"diags": []
}
self.diags = self.results["diags"]
# Make sure we have sufficient pSchedulers to cover the participants
if (nparticipants == 2) and ("pscheduler" not in z):
# TODO: Assert that Z has a host?
self.__diag("No pScheduler for or on %s." % (z["host"]))
return
# Fill in the test's blanks and construct a task spec
test = copy.deepcopy(test)
test = pscheduler.json_substitute(test, "__A__", a["pscheduler"])
z_end = z["host"] if nparticipants == 1 else z.get(
"pscheduler", z["host"])
test = pscheduler.json_substitute(test, "__Z__", z_end)
task = {
"schema": 1,
"test": test,
# This is required; empty is fine.
"schedule": {
# TODO: Don't hard-wire this.
"slip": "PT10M"
}
}
# Post the task
task_post = pscheduler.api_url(host=a["pscheduler"], path="/tasks")
status, task_href = pscheduler.url_post(
task_post, data=pscheduler.json_dump(task), throw=False)
if status != 200:
self.__diag("Unable to post task: %s" % (task_href))
return
self.__debug("Posted task %s" % (task_href))
self.task_href = task_href
# Get the task from the server with full details
status, task_data = pscheduler.url_get(task_href,
params={"detail": True},
throw=False)
if status != 200:
self.__diag("Unable to get detailed task data: %s" % (task_data))
return
# Wait for the first run to be scheduled
first_run_url = task_data["detail"]["first-run-href"]
status, run_data = pscheduler.url_get(first_run_url, throw=False)
if status == 404:
self.__diag("The server never scheduled a run for the task.")
return
if status != 200:
self.__diag("Error %d: %s" % (status, run_data))
return
for key in ["start-time", "end-time", "result-href"]:
if key not in run_data:
self.__diag("Server did not return %s with run data" % (key))
return
self.results["href"] = run_data["href"]
self.run_data = run_data
self.__debug(
"Run times: %s to %s" \
% (run_data["start-time"], run_data["end-time"]))
self.worker = threading.Thread(target=lambda: self.run())
self.worker.setDaemon(True)
self.worker.start()
def tasks():
if request.method == 'GET':
where_clause = "TRUE"
args = []
try:
json_query = arg_json("json")
except ValueError as ex:
return bad_request(str(ex))
if json_query is not None:
where_clause += " AND task.json_detail @> %s"
args.append(request.args.get("json"))
where_clause += " ORDER BY added"
tasks = __tasks_get_filtered(request.base_url,
where_clause=where_clause,
args=args,
expanded=is_expanded(),
detail=arg_boolean("detail"),
single=False)
return ok_json(tasks)
elif request.method == 'POST':
data = request.data.decode("ascii")
try:
task = pscheduler.json_load(data, max_schema=3)
except ValueError as ex:
return bad_request("Invalid task specification: %s" % (str(ex)))
# Validate the JSON against a TaskSpecification
# TODO: Figure out how to do this without the intermediate object
valid, message = pscheduler.json_validate({"": task}, {
"type": "object",
"properties": {
"": {
"$ref": "#/pScheduler/TaskSpecification"
}
},
"required": [""]
})
if not valid:
return bad_request("Invalid task specification: %s" % (message))
# See if the test spec is valid
try:
returncode, stdout, stderr = pscheduler.plugin_invoke(
"test",
task['test']['type'],
"spec-is-valid",
stdin=pscheduler.json_dump(task['test']['spec']))
if returncode != 0:
return error("Unable to validate test spec: %s" % (stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
if not validate_json["valid"]:
return bad_request(
"Invalid test specification: %s" %
(validate_json.get("error", "Unspecified error")))
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
log.debug("Validated test: %s", pscheduler.json_dump(task['test']))
# Validate the schedule
try:
cron = crontab.CronTab(task["schedule"]["repeat-cron"])
except (AttributeError, ValueError):
return error("Cron repeat specification is invalid.")
except KeyError:
pass
# Validate the archives
for archive in task.get("archives", []):
# Data
try:
returncode, stdout, stderr = pscheduler.plugin_invoke(
"archiver",
archive["archiver"],
"data-is-valid",
stdin=pscheduler.json_dump(archive["data"]),
)
if returncode != 0:
return error("Unable to validate archive spec: %s" %
(stderr))
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
try:
returned_json = pscheduler.json_load(stdout)
if not returned_json["valid"]:
return bad_request("Invalid archiver data: %s" %
(returned_json["error"]))
except Exception as ex:
return error("Internal probelm validating archiver data: %s" %
(str(ex)))
# Transform, if there was one.
if "transform" in archive:
transform = archive["transform"]
try:
_ = pscheduler.JQFilter(filter_spec=transform["script"],
args=transform.get("args", {}))
except ValueError as ex:
return error("Invalid transform: %s" % (str(ex)))
# Validate the lead binding if there was one.
lead_bind = task.get("lead-bind", None)
if lead_bind is not None \
and (pscheduler.address_interface(lead_bind) is None):
return bad_request("Lead bind '%s' is not on this host" %
(lead_bind))
# Evaluate the task against the limits and reject the request
# if it doesn't pass. We do this early so anything else in
# the process gets any rewrites.
log.debug("Checking limits on %s", task)
(processor, whynot) = limitprocessor()
if processor is None:
log.debug("Limit processor is not initialized. %s", whynot)
return no_can_do("Limit processor is not initialized: %s" % whynot)
hints, error_response = request_hints()
if hints is None:
log.debug("Can't come up with valid hints for lead task limits.")
return error_response
hints_data = pscheduler.json_dump(hints)
log.debug("Processor = %s" % processor)
passed, limits_passed, diags, new_task, _priority \
= processor.process(task, hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
if new_task is not None:
try:
task = new_task
returncode, stdout, stderr = pscheduler.plugin_invoke(
"test",
task['test']['type'],
"spec-is-valid",
stdin=pscheduler.json_dump(task["test"]["spec"]))
if returncode != 0:
return error(
"Failed to validate rewritten test specification: %s" %
(stderr))
validate_json = pscheduler.json_load(stdout, max_schema=1)
if not validate_json["valid"]:
return bad_request(
"Rewritten test specification is invalid: %s" %
(validate_json.get("error", "Unspecified error")))
except Exception as ex:
return error(
"Unable to validate rewritten test specification: " +
str(ex))
# Find the participants
try:
returncode, stdout, stderr = pscheduler.plugin_invoke(
"test",
task['test']['type'],
"participants",
stdin=pscheduler.json_dump(task['test']['spec']),
timeout=5)
if returncode != 0:
return error("Unable to determine participants: " + stderr)
participants = [
host if host is not None else server_netloc() for host in
pscheduler.json_load(stdout, max_schema=1)["participants"]
]
except Exception as ex:
return error("Exception while determining participants: " +
str(ex))
nparticipants = len(participants)
# TODO: The participants must be unique. This should be
# verified by fetching the host name from each one.
#
# TOOL SELECTION
#
# TODO: Need to provide for tool being specified by the task
# package.
tools = []
tool_params = {"test": pscheduler.json_dump(task["test"])}
tool_offers = {}
for participant_no in range(0, len(participants)):
participant = participants[participant_no]
try:
# Make sure the other participants are running pScheduler
participant_api = pscheduler.api_url_hostport(participant)
log.debug("Pinging %s" % (participant))
status, result = pscheduler.url_get(participant_api,
throw=False,
timeout=10,
bind=lead_bind)
if status == 400:
raise TaskPostingException(result)
elif status in [ 202, 204, 205, 206, 207, 208, 226,
300, 301, 302, 303, 304, 205, 306, 307, 308 ] \
or ( (status >= 400) and (status <=499) ):
raise TaskPostingException(
"Host is not running pScheduler")
elif status != 200:
raise TaskPostingException("returned status %d: %s" %
(status, result))
# TODO: This will fail with a very large test spec.
status, result = pscheduler.url_get("%s/tools" %
(participant_api),
params=tool_params,
throw=False,
bind=lead_bind)
if status != 200:
raise TaskPostingException("%d: %s" % (status, result))
tools.append(result)
except TaskPostingException as ex:
return error("Error getting tools from %s: %s" \
% (participant, str(ex)))
log.debug("Participant %s offers tools %s", participant, result)
tool_offers[participant] = result
if len(tools) != nparticipants:
return error("Didn't get a full set of tool responses")
if "tools" in task:
tool = pick_tool(tools, pick_from=task['tools'])
else:
tool = pick_tool(tools)
# Complain if no usable tool was found
if tool is None:
offers = []
for participant in participants:
participant_offers = tool_offers.get(participant,
[{
"name": "nothing"
}])
if participant_offers is not None:
offer_set = [offer["name"] for offer in participant_offers]
else:
offer_set = ["nothing"]
offers.append("%s offered %s" %
(participant, ", ".join(offer_set)))
return no_can_do("No tool in common among the participants: %s." %
("; ".join(offers)))
task['tool'] = tool
#
# TASK CREATION
#
tasks_posted = []
# Post the lead with the local database, which also assigns
# its UUID. Make it disabled so the scheduler doesn't try to
# do anything with it until the task has been submitted to all
# of the other participants.
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s, 0, %s, NULL, FALSE, %s)",
[
pscheduler.json_dump(task), participants, hints_data,
pscheduler.json_dump(limits_passed),
task.get("priority", None), diags
],
onerow=True)
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
task_uuid = cursor.fetchone()[0]
log.debug("Tasked lead, UUID %s", task_uuid)
# Other participants get the UUID and participant list forced upon them.
task["participants"] = participants
task_params = {"key": task["_key"]} if "_key" in task else {}
for participant in range(1, nparticipants):
part_name = participants[participant]
log.debug("Tasking participant %s", part_name)
try:
# Post the task
log.debug("Tasking %d@%s: %s", participant, part_name, task)
post_url = pscheduler.api_url_hostport(part_name,
'tasks/' + task_uuid)
task_params["participant"] = participant
log.debug("Posting task to %s", post_url)
status, result = pscheduler.url_post(post_url,
params=task_params,
data=task,
bind=lead_bind,
json=False,
throw=False)
log.debug("Remote returned %d: %s", status, result)
if status != 200:
raise TaskPostingException(
"Unable to post task to %s: %s" % (part_name, result))
tasks_posted.append(result)
# Fetch the task's details and add the list of limits
# passed to our own.
status, result = pscheduler.url_get(post_url,
params={"detail": True},
bind=lead_bind,
throw=False)
if status != 200:
raise TaskPostingException(
"Unable to fetch posted task from %s: %s" %
(part_name, result))
log.debug("Fetched %s", result)
try:
details = result["detail"]["spec-limits-passed"]
log.debug("Details from %s: %s", post_url, details)
limits_passed.extend(details)
except KeyError:
pass
except TaskPostingException as ex:
# Disable the task locally and let it get rid of the
# other participants.
posted_to = "%s/%s" % (request.url, task_uuid)
parsed = list(urllib.parse.urlsplit(posted_to))
parsed[1] = "%s"
template = urllib.parse.urlunsplit(parsed)
try:
dbcursor_query("SELECT api_task_disable(%s, %s)",
[task_uuid, template])
except Exception:
log.exception()
return error("Error while tasking %s: %s" % (part_name, ex))
# Update the list of limits passed in the local database
# TODO: How do the other participants know about this?
log.debug("Limits passed: %s", limits_passed)
cursor = dbcursor_query(
"UPDATE task SET limits_passed = %s::JSON WHERE uuid = %s",
[pscheduler.json_dump(limits_passed), task_uuid])
# Enable the task so the scheduler will schedule it.
try:
dbcursor_query("SELECT api_task_enable(%s)", [task_uuid])
except Exception:
log.exception()
return error("Failed to enable task %s. See system logs." %
task_uuid)
log.debug("Task enabled for scheduling.")
task_url = "%s/%s" % (request.base_url, task_uuid)
# Non-expanded gets just the URL
if not arg_boolean("expanded"):
return ok_json(task_url)
# Expanded gets a redirect to GET+expanded
params = []
for arg in ["detail", "pretty"]:
if arg_boolean(arg):
params.append(arg)
if params:
task_url += "?%s" % ("&".join(params))
return see_other(task_url)
else:
return not_allowed()
def tasks_uuid_runs_run_result(task, run):
if task is None:
return bad_request("Missing or invalid task")
if run is None:
return bad_request("Missing or invalid run")
wait = arg_boolean('wait')
format = request.args.get('format')
if format is None:
format = 'application/json'
if format not in [ 'application/json', 'text/html', 'text/plain' ]:
return bad_request("Unsupported format " + format)
# If asked for 'first', dig up the first run and use its UUID.
# This is more for debug convenience than anything else.
if run == 'first':
try:
run = __runs_first_run(task)
except Exception as ex:
log.exception()
return error(str(ex))
if run is None:
return not_found()
#
# Camp on the run for a result
#
# 40 tries at 0.25s intervals == 10 sec.
tries = 40 if wait else 1
while tries:
try:
cursor = dbcursor_query("""
SELECT
test.name,
run.result_merged,
task.json #> '{test, spec}'
FROM
run
JOIN task ON task.id = run.task
JOIN test ON test.id = task.test
WHERE
task.uuid = %s
AND run.uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
# TODO: Make sure we got back one row with two columns.
row = cursor.fetchone()
if not wait and row[1] is None:
time.sleep(0.25)
tries -= 1
else:
break
if tries == 0:
return not_found()
test_type, merged_result, test_spec = row
# JSON requires no formatting.
if format == 'application/json':
return ok_json(merged_result)
if not merged_result['succeeded']:
if format == 'text/plain':
return ok("Test failed.", mimetype=format)
elif format == 'text/html':
return ok("<p>Test failed.</p>", mimetype=format)
return error("Unsupported format " + format)
formatter_input = {
"spec": test_spec,
"result": merged_result
}
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", test_type,
"result-format", format ],
stdin = pscheduler.json_dump(formatter_input)
)
if returncode != 0:
return error("Failed to format result: " + stderr)
return ok(stdout.rstrip(), mimetype=format)
def __init__(self, calls, argv=[], stdin=None):
"""The 'calls' argument is an array of dictionaries. Each dictionary
contains an entry called "call," a string that names the program
to be run, and another called "input" which is an arbitrary blob
of data (usually a dictionary) that can be converted into JSON and
passed through to the program's standard input.
The format for the program's input is that for a pScheduler
context plugin's "change" method. It consists of two items:
"data", which is an arbitrary blob of (JSON) data for the
program to use and "exec," a string indicating the path to the
program that should be exec'd when the program completes
successfully.
For example:
{
"program": "/run/this/program",
"input": {
"data": { "foo": "bar", "baz": 31415 },
"exec": "/run/that/program"
}
}
The "argv" and "stdin" are program parameters and standard
input with the same semantics as the same arguments to
pscheduler.run_program().
"""
if not isinstance(calls, list):
raise ValueError("Calls must be a list.")
self.stages = []
if not calls:
return
# Create the temporary files that will hold the scripts.
try:
# Create a list of temporary files ahead of time so the
# stage n script can refer to the one for stage n+1. The
# extra added on is the "final" stage where the program to
# be run is actually run.
for _ in range(0, len(calls) + 1):
(fileno, path) = tempfile.mkstemp(prefix="ContextedRunner-")
os.close(fileno)
os.chmod(path, stat.S_IRWXU)
self.stages.append(path)
# Write the scripts
for stage in range(0, len(calls)):
stage_script = "#!/bin/sh -e\n" \
"exec %s <<'EOF'\n" \
"%s\n" \
"EOF\n" % (
" ".join([quote(arg) for arg in calls[stage]["program"]]),
pscheduler.json_dump({
"data": calls[stage]["input"],
"exec": self.stages[stage+1]
}))
with open(self.stages[stage], "w") as output:
output.write(stage_script)
# Write the "final" stage
with open(self.stages[-1], "w") as output:
output.write("#!/bin/sh -e\n"
"exec %s" %
(" ".join([quote(arg) for arg in argv])))
if stdin is not None:
output.write(" <<'EOF'\n"
"%s%s"
"EOF\n" %
(stdin, "" if stdin[-1] == "\n" else "\n"))
else:
# PORT: This is Unix-specific.
output.write(" < /dev/null\n")
except Exception as ex:
for remove in self.stages:
try:
os.unlink(remove)
except IOError:
pass # This is best effort only.
raise ex
def response_json_dump(dump, sanitize=True):
if sanitize:
sanitized = pscheduler.json_decomment(dump, prefix="_", null=True)
return pscheduler.json_dump(sanitized, pretty=arg_boolean('pretty'))
else:
return pscheduler.json_dump(dump, pretty=arg_boolean('pretty'))
def tasks_uuid_runs_run(task, run):
if not uuid_is_valid(task):
return not_found()
if ((request.method in ['PUT', 'DELETE'] and not uuid_is_valid(run))
or (run not in ['first', 'next'] and not uuid_is_valid(run))):
return not_found()
if request.method == 'GET':
# Wait for there to be a local result
wait_local = arg_boolean('wait-local')
# Wait for there to be a merged result
wait_merged = arg_boolean('wait-merged')
if wait_local and wait_merged:
return bad_request("Cannot wait on local and merged results")
# Figure out how long to wait in seconds. Zero means don't
# wait.
wait_time = arg_integer('wait')
if wait_time is None:
wait_time = 30
if wait_time < 0:
return bad_request("Wait time must be >= 0")
# If asked for 'first', dig up the first run and use its UUID.
if run in ['next', 'first']:
future = run == 'next'
wait_interval = 0.5
tries = int(wait_time / wait_interval) if wait_time > 0 \
else 1
while tries > 0:
run = __runs_first_run(task, future)
if run is not None:
break
if wait_time > 0:
time.sleep(1.0)
tries -= 1
if run is None:
return not_found()
# Obey the wait time with tries at 0.5s intervals
tries = wait_time * 2 if (wait_local or wait_merged) else 1
result = {}
while tries:
try:
cursor = dbcursor_query(
"""
SELECT
run_json(run.id),
run_state.finished
FROM
task
JOIN run ON task.id = run.task
JOIN run_state ON run_state.id = run.state
WHERE
task.uuid = %s
AND run.uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
cursor.close()
return not_found()
result, finished = cursor.fetchone()
cursor.close()
if not (wait_local or wait_merged):
break
else:
if (wait_local and result['result'] is None) \
or (wait_merged \
and ( (result['result-full'] is None) or (not finished) ) ):
log.debug("Waiting (%d left) for merged: %s %s", tries,
result['result-full'], finished)
time.sleep(0.5)
tries -= 1
else:
log.debug("Got the requested result.")
break
# Even if we timed out waiting, return the last result we saw
# and let the client sort it out.
# This strips any query parameters and replaces the last item
# with the run, which might be needed if the 'first' option
# was used.
href_path_parts = urllib.parse.urlparse(request.url).path.split('/')
href_path_parts[-1] = run
href_path = '/'.join(href_path_parts)
href = urllib.parse.urljoin(request.url, href_path)
result['href'] = href
result['task-href'] = root_url('tasks/' + task)
result['result-href'] = href + '/result'
# For a NULL first participant, fill in the netloc.
try:
if result['participants'][0] is None:
result['participants'][0] = server_netloc()
except KeyError:
pass # Not there? Don't care.
return json_response(result)
elif request.method == 'PUT':
data = request.data.decode("ascii")
log.debug("Run PUT %s", request.url)
requester, key = task_requester_key(task)
if requester is None:
return not_found()
if not access_write_task(requester, key):
return forbidden()
# Get the JSON from the body
try:
run_data = pscheduler.json_load(data, max_schema=1)
except ValueError:
log.exception()
log.debug("Run data was %s", data)
return bad_request("Invalid or missing run data")
# If the run doesn't exist, take the whole thing as if it were
# a POST.
cursor = dbcursor_query(
"SELECT EXISTS (SELECT * FROM run WHERE uuid = %s)", [run],
onerow=True)
fetched = cursor.fetchone()[0]
cursor.close()
if not fetched:
log.debug("Record does not exist; full PUT.")
try:
start_time = \
pscheduler.iso8601_as_datetime(run_data['start-time'])
except KeyError:
return bad_request("Missing start time")
except ValueError:
return bad_request("Invalid start time")
try:
passed, diags, response, priority \
= __evaluate_limits(task, start_time)
if response is not None:
return response
if passed:
diag_message = None
else:
diag_message = "Run forbidden by limits:\n%s" % (diags)
cursor = dbcursor_query(
"SELECT * FROM api_run_post(%s, %s, %s, %s, %s, %s)",
[task, start_time, run, diag_message, priority, diags],
onerow=True)
succeeded, uuid, conflicts, error_message = cursor.fetchone()
cursor.close()
if conflicts:
return conflict(error_message)
if not succeeded:
return error(error_message)
log.debug("Full put of %s, got back %s", run, uuid)
except Exception as ex:
log.exception()
return error(str(ex))
return ok()
# For anything else, only one thing can be udated at a time,
# and even that is a select subset.
log.debug("Record exists; partial PUT.")
if 'part-data-full' in run_data:
log.debug("Updating part-data-full from %s", run_data)
try:
part_data_full = \
pscheduler.json_dump(run_data['part-data-full'])
except KeyError:
return bad_request("Missing part-data-full")
except ValueError:
return bad_request("Invalid part-data-full")
log.debug("Full data is: %s", part_data_full)
cursor = dbcursor_query(
"""
UPDATE
run
SET
part_data_full = %s
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""", [part_data_full, run, task])
rowcount = cursor.rowcount
cursor.close()
if rowcount != 1:
return not_found()
log.debug("Full data updated")
return ok()
elif 'result-full' in run_data:
log.debug("Updating result-full from %s", run_data)
try:
result_full = \
pscheduler.json_dump(run_data['result-full'])
except KeyError:
return bad_request("Missing result-full")
except ValueError:
return bad_request("Invalid result-full")
try:
succeeded = bool(run_data['succeeded'])
except KeyError:
return bad_request("Missing success value")
except ValueError:
return bad_request("Invalid success value")
log.debug("Updating result-full: JSON %s", result_full)
log.debug("Updating result-full: Run %s", run)
log.debug("Updating result-full: Task %s", task)
cursor = dbcursor_query(
"""
UPDATE
run
SET
result_full = %s,
state = CASE %s
WHEN TRUE THEN run_state_finished()
ELSE run_state_failed()
END
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""", [result_full, succeeded, run, task])
rowcount = cursor.rowcount
cursor.close()
if rowcount != 1:
return not_found()
return ok()
elif request.method == 'DELETE':
# TODO: If this is the lead, the run's counterparts on the
# other participating nodes need to be removed as well.
requester, key = task_requester_key(task)
if requester is None:
return not_found()
if not access_write_task(requester, key):
return forbidden()
cursor = dbcursor_query(
"""
DELETE FROM run
WHERE
task in (SELECT id FROM task WHERE uuid = %s)
AND uuid = %s
""", [task, run])
rowcount = cursor.rowcount
cursor.close()
return ok() if rowcount == 1 else not_found()
else:
return not_allowed()
raw_offset = time_here.strftime("%z")
if len(raw_offset):
offset = raw_offset[:3] + ":" + raw_offset[-2:]
else:
offset = ""
result = {"time": time_here.strftime("%Y-%m-%dT%H:%M:%S.%f") + offset, "synchronized": system_synchronized}
if system_synchronized:
# Assume NTP for the time being
try:
ntp = ntplib.NTPClient().request("127.0.0.1")
result["offset"] = ntp.offset
result["source"] = "ntp"
result["reference"] = "%s from %s" % (
ntplib.stratum_to_text(ntp.stratum),
ntplib.ref_id_to_text(ntp.ref_id),
)
except Exception as ex:
result["error"] = str(ex)
return result
if __name__ == "__main__":
import pscheduler
print pscheduler.json_dump(clock_state(), pretty=True)
def tasks_uuid_runs_run_result(task, run):
if not uuid_is_valid(task) or not uuid_is_valid(run):
return not_found()
wait = arg_boolean('wait')
format = request.args.get('format')
if format is None:
format = 'application/json'
if format not in ['application/json', 'text/html', 'text/plain']:
return bad_request("Unsupported format " + format)
# If asked for 'first', dig up the first run and use its UUID.
# This is more for debug convenience than anything else.
if run == 'first':
run = __runs_first_run(task)
if run is None:
return not_found()
#
# Camp on the run for a result
#
# 40 tries at 0.25s intervals == 10 sec.
tries = 40 if wait else 1
while tries:
cursor = dbcursor_query(
"""
SELECT
test.name,
run.result_merged,
task.json #> '{test, spec}'
FROM
run
JOIN task ON task.id = run.task
JOIN test ON test.id = task.test
WHERE
task.uuid = %s
AND run.uuid = %s
""", [task, run])
if cursor.rowcount == 0:
cursor.close()
return not_found()
# TODO: Make sure we got back one row with two columns.
row = cursor.fetchone()
cursor.close()
if not wait and row[1] is None:
time.sleep(0.25)
tries -= 1
else:
break
if tries == 0:
return not_found()
test_type, merged_result, test_spec = row
# JSON requires no formatting.
if format == 'application/json':
return ok_json(merged_result)
if not merged_result['succeeded']:
if format == 'text/plain':
return ok("Run failed.", mimetype=format)
elif format == 'text/html':
return ok("<p>Run failed.</p>", mimetype=format)
return bad_request("Unsupported format " + format)
formatter_input = {"spec": test_spec, "result": merged_result}
returncode, stdout, stderr = pscheduler.plugin_invoke(
"test",
test_type,
"result-format",
argv=[format],
stdin=pscheduler.json_dump(formatter_input))
if returncode != 0:
return error("Failed to format result: " + stderr)
return ok(stdout.rstrip(), mimetype=format)
def tasks_uuid_runs_run(task, run):
if not uuid_is_valid(task):
return not_found()
if ((request.method in ['PUT', 'DELETE'] and not uuid_is_valid(run))
or (run not in ['first', 'next'] and not uuid_is_valid(run))):
return not_found()
if request.method == 'GET':
# Wait for there to be a local result
wait_local = arg_boolean('wait-local')
# Wait for there to be a merged result
wait_merged = arg_boolean('wait-merged')
if wait_local and wait_merged:
return bad_request("Cannot wait on local and merged results")
# Figure out how long to wait in seconds. Zero means don't
# wait.
wait_time = arg_integer('wait')
if wait_time is None:
wait_time = 30
if wait_time < 0:
return bad_request("Wait time must be >= 0")
# If asked for 'first', dig up the first run and use its UUID.
if run in ['next', 'first']:
future = run == 'next'
wait_interval = 0.5
tries = int(wait_time / wait_interval) if wait_time > 0 \
else 1
while tries > 0:
try:
run = __runs_first_run(task, future)
except Exception as ex:
log.exception()
return error(str(ex))
if run is not None:
break
if wait_time > 0:
time.sleep(1.0)
tries -= 1
if run is None:
return not_found()
# 60 tries at 0.5s intervals == 30 sec.
tries = 60 if (wait_local or wait_merged) else 1
while tries:
try:
cursor = dbcursor_query(
"""
SELECT
lower(run.times),
upper(run.times),
upper(run.times) - lower(run.times),
task.participant,
task.nparticipants,
task.participants,
run.part_data,
run.part_data_full,
run.result,
run.result_full,
run.result_merged,
run_state.enum,
run_state.display,
run.errors,
run.clock_survey,
run.id,
archiving_json(run.id),
run.added
FROM
run
JOIN task ON task.id = run.task
JOIN run_state ON run_state.id = run.state
WHERE
task.uuid = %s
AND run.uuid = %s""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
cursor.close()
return not_found()
row = cursor.fetchone()
cursor.close()
if not (wait_local or wait_merged):
break
else:
if (wait_local and row[7] is None) \
or (wait_merged and row[9] is None):
time.sleep(0.5)
tries -= 1
else:
break
# Return a result Whether or not we timed out and let the
# client sort it out.
result = {}
# This strips any query parameters and replaces the last item
# with the run, which might be needed if the 'first' option
# was used.
href_path_parts = urlparse.urlparse(request.url).path.split('/')
href_path_parts[-1] = run
href_path = '/'.join(href_path_parts)
href = urlparse.urljoin(request.url, href_path)
result['href'] = href
result['start-time'] = pscheduler.datetime_as_iso8601(row[0])
result['end-time'] = pscheduler.datetime_as_iso8601(row[1])
result['duration'] = pscheduler.timedelta_as_iso8601(row[2])
participant_num = row[3]
result['participant'] = participant_num
result['participants'] = [
server_netloc()
if participant is None and participant_num == 0 else participant
for participant in row[5]
]
result['participant-data'] = row[6]
result['participant-data-full'] = row[7]
result['result'] = row[8]
result['result-full'] = row[9]
result['result-merged'] = row[10]
result['state'] = row[11]
result['state-display'] = row[12]
result['errors'] = row[13]
if row[14] is not None:
result['clock-survey'] = row[14]
if row[16] is not None:
result['archivings'] = row[16]
if row[17] is not None:
result['added'] = pscheduler.datetime_as_iso8601(row[17])
result['task-href'] = root_url('tasks/' + task)
result['result-href'] = href + '/result'
return json_response(result)
elif request.method == 'PUT':
log.debug("Run PUT %s", request.url)
# Get the JSON from the body
try:
run_data = pscheduler.json_load(request.data, max_schema=1)
except ValueError:
log.exception()
log.debug("Run data was %s", request.data)
return bad_request("Invalid or missing run data")
# If the run doesn't exist, take the whole thing as if it were
# a POST.
try:
cursor = dbcursor_query(
"SELECT EXISTS (SELECT * FROM run WHERE uuid = %s)", [run],
onerow=True)
except Exception as ex:
log.exception()
return error(str(ex))
fetched = cursor.fetchone()[0]
cursor.close()
if not fetched:
log.debug("Record does not exist; full PUT.")
try:
start_time = \
pscheduler.iso8601_as_datetime(run_data['start-time'])
except KeyError:
return bad_request("Missing start time")
except ValueError:
return bad_request("Invalid start time")
try:
passed, diags, response = __evaluate_limits(task, start_time)
if response is not None:
return response
cursor = dbcursor_query(
"SELECT * FROM api_run_post(%s, %s, %s)",
[task, start_time, run],
onerow=True)
succeeded, uuid, conflicts, error_message = cursor.fetchone()
cursor.close()
if conflicts:
return conflict(error_message)
if not succeeded:
return error(error_message)
log.debug("Full put of %s, got back %s", run, uuid)
except Exception as ex:
log.exception()
return error(str(ex))
return ok()
# For anything else, only one thing can be udated at a time,
# and even that is a select subset.
log.debug("Record exists; partial PUT.")
if 'part-data-full' in run_data:
log.debug("Updating part-data-full from %s", run_data)
try:
part_data_full = \
pscheduler.json_dump(run_data['part-data-full'])
except KeyError:
return bad_request("Missing part-data-full")
except ValueError:
return bad_request("Invalid part-data-full")
log.debug("Full data is: %s", part_data_full)
try:
cursor = dbcursor_query(
"""
UPDATE
run
SET
part_data_full = %s
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""", [part_data_full, run, task])
except Exception as ex:
log.exception()
return error(str(ex))
rowcount = cursor.rowcount
cursor.close()
if rowcount != 1:
return not_found()
log.debug("Full data updated")
return ok()
elif 'result-full' in run_data:
log.debug("Updating result-full from %s", run_data)
try:
result_full = \
pscheduler.json_dump(run_data['result-full'])
except KeyError:
return bad_request("Missing result-full")
except ValueError:
return bad_request("Invalid result-full")
try:
succeeded = bool(run_data['succeeded'])
except KeyError:
return bad_request("Missing success value")
except ValueError:
return bad_request("Invalid success value")
log.debug("Updating result-full: JSON %s", result_full)
log.debug("Updating result-full: Run %s", run)
log.debug("Updating result-full: Task %s", task)
try:
cursor = dbcursor_query(
"""
UPDATE
run
SET
result_full = %s,
state = CASE %s
WHEN TRUE THEN run_state_finished()
ELSE run_state_failed()
END
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""", [result_full, succeeded, run, task])
except Exception as ex:
log.exception()
return error(str(ex))
rowcount = cursor.rowcount
cursor.close()
if rowcount != 1:
return not_found()
return ok()
elif request.method == 'DELETE':
# TODO: If this is the lead, the run's counterparts on the
# other participating nodes need to be removed as well.
try:
requester = task_requester(task)
if requester is None:
return not_found()
if not access_write_ok(requester):
return forbidden()
except Exception as ex:
return error(str(ex))
try:
cursor = dbcursor_query(
"""
DELETE FROM run
WHERE
task in (SELECT id FROM task WHERE uuid = %s)
AND uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
rowcount = cursor.rowcount
cursor.close()
return ok() if rowcount == 1 else not_found()
else:
return not_allowed()
def tasks_uuid(uuid):
if request.method == 'GET':
# Get a task, adding server-derived details if a 'detail'
# argument is present.
try:
cursor = dbcursor_query("""
SELECT
task.json,
task.added,
task.start,
task.slip,
task.duration,
task.post,
task.runs,
task.participants,
scheduling_class.anytime,
scheduling_class.exclusive,
scheduling_class.multi_result,
task.participant,
task.enabled,
task.cli
FROM
task
JOIN test ON test.id = task.test
JOIN scheduling_class
ON scheduling_class.id = test.scheduling_class
WHERE uuid = %s
""", [uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if row is None:
return not_found()
json = row[0]
# Redact anything in the test spec or archivers that's marked
# private as well as _key at the top level if there is one.
if "_key" in json:
json["_key"] = None
json["test"]["spec"] = pscheduler.json_decomment(
json["test"]["spec"], prefix="_", null=True)
try:
for archive in range(0,len(json["archives"])):
json["archives"][archive]["data"] = pscheduler.json_decomment(
json["archives"][archive]["data"], prefix="_", null=True)
except KeyError:
pass # Don't care if not there.
# Add details if we were asked for them.
if arg_boolean('detail'):
part_list = row[7];
# The database is not supposed to allow this, but spit out
# a sane default as a last resort in case it happens.
if part_list is None:
part_list = [None]
if row[10] == 0 and part_list[0] is None:
part_list[0] = pscheduler.api_this_host()
json['detail'] = {
'added': None if row[1] is None \
else pscheduler.datetime_as_iso8601(row[1]),
'start': None if row[2] is None \
else pscheduler.datetime_as_iso8601(row[2]),
'slip': None if row[3] is None \
else pscheduler.timedelta_as_iso8601(row[3]),
'duration': None if row[4] is None \
else pscheduler.timedelta_as_iso8601(row[4]),
'post': None if row[5] is None \
else pscheduler.timedelta_as_iso8601(row[5]),
'runs': None if row[6] is None \
else int(row[6]),
'participants': part_list,
'anytime': row[8],
'exclusive': row[9],
'multi-result': row[10],
'enabled': row[12],
'cli': row[13]
}
return ok_json(json)
elif request.method == 'POST':
log.debug("Posting to %s", uuid)
log.debug("Data is %s", request.data)
# TODO: This is only for participant 1+
# TODO: This should probably a PUT and not a POST.
try:
json_in = pscheduler.json_load(request.data)
except ValueError:
return bad_request("Invalid JSON")
log.debug("JSON is %s", json_in)
try:
participant = arg_cardinal('participant')
except ValueError as ex:
return bad_request("Invalid participant: " + str(ex))
log.debug("Participant %d", participant)
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on task")
processor, whynot = limitprocessor()
if processor is None:
message = "Limit processor is not initialized: %s" % whynot
log.debug(message)
return no_can_do(message)
# TODO: This is cooked up in two places. Make a function of it.
hints = {
"ip": request.remote_addr
}
hints_data = pscheduler.json_dump(hints)
passed, diags = processor.process(json_in["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
log.debug("Limits passed")
# TODO: Pluck UUID from URI
uuid = url_last_in_path(request.url)
log.debug("Posting task %s", uuid)
try:
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s)",
[request.data, hints_data, participant, uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
# TODO: Assert that rowcount is 1
log.debug("All done: %s", base_url())
return ok(base_url())
elif request.method == 'DELETE':
parsed = list(urlparse.urlsplit(request.url))
parsed[1] = "%s"
template = urlparse.urlunsplit(parsed)
try:
cursor = dbcursor_query(
"SELECT api_task_disable(%s, %s)", [uuid, template])
except Exception as ex:
return error(str(ex))
return ok()
else:
return not_allowed()
self.__debug("Worker started")
try:
self.__run()
except Exception as ex:
self.__diag(str(ex))
def result(self):
"""Wait for the result and return it."""
try:
if self.worker.is_alive():
self.worker.join()
except AttributeError:
pass # Don't care if it's not there.
return self.results
if __name__ == "__main__":
test = {"type": "simplestream", "spec": {"schema": 1, "dest": "__Z__"}}
a = {"pscheduler": "dev7", "host": "dev7"}
z = {"pscheduler": "dev6", "host": "dev6"}
nparticipants = 2
r = TaskRunner(test, nparticipants, a, z, debug=True)
print pscheduler.json_dump(r.result(), pretty=True)
def tasks_uuid_runs_run(task, run):
if task is None:
return bad_request("Missing or invalid task")
if run is None:
return bad_request("Missing or invalid run")
if request.method == 'GET':
# Wait for there to be a local result
wait_local = arg_boolean('wait-local')
# Wait for there to be a merged result
wait_merged = arg_boolean('wait-merged')
if wait_local and wait_merged:
return error("Cannot wait on local and merged results")
# If asked for 'first', dig up the first run and use its UUID.
if run == 'first':
# 60 tries at 0.5s intervals == 30 sec.
tries = 60
while tries > 0:
try:
run = __runs_first_run(task)
except Exception as ex:
log.exception()
return error(str(ex))
if run is not None:
break
time.sleep(1.0)
tries -= 1
if run is None:
return not_found()
# 60 tries at 0.5s intervals == 30 sec.
tries = 60 if (wait_local or wait_merged) else 1
while tries:
try:
cursor = dbcursor_query(
"""
SELECT
lower(run.times),
upper(run.times),
upper(run.times) - lower(run.times),
task.participant,
task.nparticipants,
task.participants,
run.part_data,
run.part_data_full,
run.result,
run.result_full,
run.result_merged,
run_state.enum,
run_state.display,
run.errors,
run.clock_survey
FROM
run
JOIN task ON task.id = run.task
JOIN run_state ON run_state.id = run.state
WHERE
task.uuid = %s
AND run.uuid = %s""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if not (wait_local or wait_merged):
break
else:
if (wait_local and row[7] is None) \
or (wait_merged and row[9] is None):
time.sleep(0.5)
tries -= 1
else:
break
# Return a result Whether or not we timed out and let the
# client sort it out.
result = {}
# This strips any query parameters and replaces the last item
# with the run, which might be needed if the 'first' option
# was used.
href_path_parts = urlparse.urlparse(request.url).path.split('/')
href_path_parts[-1] = run
href_path = '/'.join(href_path_parts)
href = urlparse.urljoin( request.url, href_path )
result['href'] = href
result['start-time'] = pscheduler.datetime_as_iso8601(row[0])
result['end-time'] = pscheduler.datetime_as_iso8601(row[1])
result['duration'] = pscheduler.timedelta_as_iso8601(row[2])
participant_num = row[3]
result['participant'] = participant_num
result['participants'] = [
pscheduler.api_this_host()
if participant is None and participant_num == 0
else participant
for participant in row[5]
]
result['participant-data'] = row[6]
result['participant-data-full'] = row[7]
result['result'] = row[8]
result['result-full'] = row[9]
result['result-merged'] = row[10]
result['state'] = row[11]
result['state-display'] = row[12]
result['errors'] = row[13]
if row[14] is not None:
result['clock-survey'] = row[14]
result['task-href'] = root_url('tasks/' + task)
result['result-href'] = href + '/result'
return json_response(result)
elif request.method == 'PUT':
log.debug("Run PUT %s", request.url)
# Get the JSON from the body
try:
run_data = pscheduler.json_load(request.data)
except ValueError:
log.exception()
log.debug("Run data was %s", request.data)
return error("Invalid or missing run data")
# If the run doesn't exist, take the whole thing as if it were
# a POST.
try:
cursor = dbcursor_query(
"SELECT EXISTS (SELECT * FROM run WHERE uuid = %s)",
[run], onerow=True)
except Exception as ex:
log.exception()
return error(str(ex))
if not cursor.fetchone()[0]:
log.debug("Record does not exist; full PUT.")
try:
start_time = \
pscheduler.iso8601_as_datetime(run_data['start-time'])
except KeyError:
return bad_request("Missing start time")
except ValueError:
return bad_request("Invalid start time")
passed, diags = __evaluate_limits(task, start_time)
try:
cursor = dbcursor_query("SELECT api_run_post(%s, %s, %s)",
[task, start_time, run], onerow=True)
log.debug("Full put of %s, got back %s", run, cursor.fetchone()[0])
except Exception as ex:
log.exception()
return error(str(ex))
return ok()
# For anything else, only one thing can be udated at a time,
# and even that is a select subset.
log.debug("Record exists; partial PUT.")
if 'part-data-full' in run_data:
log.debug("Updating part-data-full from %s", run_data)
try:
part_data_full = \
pscheduler.json_dump(run_data['part-data-full'])
except KeyError:
return bad_request("Missing part-data-full")
except ValueError:
return bad_request("Invalid part-data-full")
log.debug("Full data is: %s", part_data_full)
try:
cursor = dbcursor_query("""
UPDATE
run
SET
part_data_full = %s
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""",
[ part_data_full, run, task])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount != 1:
return not_found()
log.debug("Full data updated")
return ok()
elif 'result-full' in run_data:
log.debug("Updating result-full from %s", run_data)
try:
result_full = \
pscheduler.json_dump(run_data['result-full'])
except KeyError:
return bad_request("Missing result-full")
except ValueError:
return bad_request("Invalid result-full")
try:
succeeded = bool(run_data['succeeded'])
except KeyError:
return bad_request("Missing success value")
except ValueError:
return bad_request("Invalid success value")
log.debug("Updating result-full: JSON %s", result_full)
log.debug("Updating result-full: Run %s", run)
log.debug("Updating result-full: Task %s", task)
try:
cursor = dbcursor_query("""
UPDATE
run
SET
result_full = %s,
state = CASE %s
WHEN TRUE THEN run_state_finished()
ELSE run_state_failed()
END
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""",
[ result_full, succeeded, run, task ])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount != 1:
return not_found()
return ok()
elif request.method == 'DELETE':
# TODO: If this is the lead, the run's counterparts on the
# other participating nodes need to be removed as well.
try:
cursor = dbcursor_query("""
DELETE FROM run
WHERE
task in (SELECT id FROM task WHERE uuid = %s)
AND uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
return ok() if cursor.rowcount == 1 else not_found()
else:
return not_allowed()
},
"parting-comment": {
"description":
"Parting comment must contain a vowel if not empty",
"match": {
"style": "regex",
"match": "(^$|[aeiou])",
"case-insensitive": True
}
}
}
})
print pscheduler.json_dump(limit.evaluate({
"task": {
"test": {
"type": "idle",
"spec": {
"schema": 1,
"#duration": "PT45M",
"duration": "PT45S",
"starting-comment": "Perry the PLATYPUS",
"#starting-comment": "Ferb",
"#parting-comment": "Vwl!",
"parting-comment": "Vowel!"
}
}
}
}),
pretty=True)
def __call__(self, json):
"""Emit serialized JSON to the file"""
self.emit_text(pscheduler.json_dump(json, pretty=False))
"type": "http",
"spec": {
"url": "https://www.not-a-real-domain.foo/",
"parse": "perfSONAR",
"keep-content": 100,
"always-succeed": False
}
}
}, {
"test": {
"type": "http",
"spec": {
"url": "https://www.not-a-real-domain.foo/",
"parse": "perfSONAR",
"keep-content": 100,
"always-succeed": True
}
}
}, {
"test": {
"type": "http",
"spec": {
"url": "https://www.perfsonar.net",
"parse": "perfSONAR",
"keep-content": 100,
"always-succeed": True
}
}
}]:
print(pscheduler.json_dump(run(data), pretty=True))
def json_dump(dump):
return pscheduler.json_dump(dump,
pretty=arg_boolean('pretty')
)
"schedule": {
"max-runs": 3,
"repeat": "PT10S",
"slip": "PT5M"
},
"test": {
"spec": {
"dest": "www.notonthe.net",
"schema": 1
},
"type": "trace"
},
"tools": ["traceroute", "paris-traceroute"]
}
try:
(changed, new_task, diags) = rewriter(task, ["c1", "c2", "c3"])
if changed:
if len(diags):
print "Diagnostics:"
print "\n".join(map(lambda s: " - " + s, diags))
else:
print "No diagnostics."
print
print pscheduler.json_dump(new_task, pretty=True)
else:
print "No changes."
except Exception as ex:
print "Failed:", ex
