def __init__(self, source=None): # Open file or path to file with limit set
"""Construct a limit processor.
The 'source' argument can be one of three types:
string - Path to a file to open and read.
file - Open file handle to read.
None - The limit processor becomes inert and passes all limits
unconditionally.
"""
# If we were given no source, put the processor into an inert
# mode where everything passes.
self.inert = source is None
if self.inert:
return
#
# Load the validation data
#
validation_path = os.path.join(os.path.dirname(__file__), "pscheduler-limits-validate.json")
# TODO: Throw something nicer than IOError if this fails.
validation_file = open(validation_path, "r")
validation = pscheduler.json_load(validation_file)
validation_file.close()
#
# Inhale the source and validate it
#
if type(source) is str or type(source) is unicode:
source = open(source, "r")
elif type(source) is file:
pass # We're good with this.
else:
raise ValueError("Source must be a file or path")
# At this point, source is a file.
assert type(source) is file
limit_config = pscheduler.json_load(source)
valid, message = pscheduler.json_validate(limit_config, validation)
if not valid:
raise ValueError("Invalid limit file: %s" % message)
#
# Set up all of the stages
#
self.identifiers = IdentifierSet(limit_config["identifiers"])
self.classifiers = ClassifierSet(limit_config["classifiers"], self.identifiers)
self.limits = LimitSet(limit_config["limits"])
self.applications = ApplicationSet(limit_config["applications"], self.classifiers, self.limits)
def evaluate(self, run): # The proposed run
# Dissent if the test isn't our type
if run["type"] != self.test:
return {"passed": False, "reasons": ["Test is not '%s'" % self.test]}
pass_input = {"spec": run["spec"], "limit": self.limit}
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", self.test, "limit-passes"],
stdin=pscheduler.json_dump(pass_input),
# TODO: Is this reasonable?
timeout=5,
)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
check_result = pscheduler.json_load(stdout)
passed = check_result["passes"]
result = {"passed": passed}
if not passed:
result["reasons"] = check_result["errors"]
return result
def arg_json(name):
"""Fetch and validate an argument as JSON"""
argval = request.args.get(name)
if argval is None:
return None
# This will throw a ValueError if something's wrong.
return pscheduler.json_load(argval)
def tests_name_spec(name):
try:
cursor = dbcursor_query("SELECT EXISTS (SELECT * FROM test WHERE NAME = %s)",
[ name ])
except Exception as ex:
return error(str(ex))
exists = cursor.fetchone()[0]
if not exists:
return not_found()
try:
args = arg_json('args')
except ValueError:
return error("Invalid JSON passed to 'args'")
status, stdout, stderr = pscheduler.run_program(
[ 'pscheduler', 'internal', 'invoke', 'test', name, 'cli-to-spec' ],
stdin = pscheduler.json_dump(args),
short = True,
)
if status != 0:
return bad_request(stderr)
# The extra parse here makes 'pretty' work.
returned_json = pscheduler.json_load(stdout)
return ok_json(returned_json)
def create_metadata(self, metadata):
result = {}
post_url = self.url
if not post_url.endswith('/'):
post_url += '/'
log.debug("Posting metadata to %s: %s" % (post_url, metadata))
r = requests.post(post_url, data=pscheduler.json_dump(metadata), headers=self.headers, verify=self.verify_ssl)
if r.status_code != 200 and r.status_code != 201:
try:
return False, "%d: %s" % (r.status_code, pscheduler.json_load(r.text)['detail'])
except:
return False, "%d: %s" % (r.status_code, r.text)
try:
rjson = pscheduler.json_load(r.text)
log.debug("Metadata POST result: %s" % rjson)
except:
return False, "Invalid JSON returned from server: %s" % r.text
return True, rjson
def create_data(self, metadata_key, data_points):
result = {}
put_url = self.url
if not put_url.endswith('/'):
put_url += '/'
put_url += ("%s/" % metadata_key)
data = { 'data': data_points }
log.debug("Putting data to %s: %s" % (put_url, data))
r = requests.put(put_url, data=pscheduler.json_dump(data), headers=self.headers, verify=self.verify_ssl)
if r.status_code== 409:
#duplicate data
log.debug("Attempted to add duplicate data point. Skipping")
elif r.status_code != 200 and r.status_code != 201:
try:
return False, "%d: %s" % (r.status_code, pscheduler.json_load(r.text)['detail'])
except:
return False, "%d: %s" % (r.status_code, r.text)
return True, ""
def __init__(self, data): # Data suitable for this class
valid, message = test_data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.test = data["test"]
self.limit = data["limit"]
returncode, stdout, stderr = pscheduler.run_program(
["pscheduler", "internal", "invoke", "test", self.test, "limit-is-valid"],
stdin=pscheduler.json_dump(self.limit),
# TODO: Is this reasonable?
timeout=5,
)
if returncode != 0:
raise RuntimeError("Failed to validate limit: %s" % stderr)
result = pscheduler.json_load(stdout)
if not result["valid"]:
raise ValueError("Invalid limit: %s" % result["message"])
def tasks_uuid_cli(uuid):
# Get a task, adding server-derived details if a 'detail'
# argument is present.
try:
cursor = dbcursor_query(
"""SELECT
task.json #>> '{test, spec}',
test.name
FROM
task
JOIN test on test.id = task.test
WHERE task.uuid = %s""", [uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if row is None:
return not_found()
json, test = row
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
test, "spec-to-cli" ], stdin = json )
if returncode != 0:
return error("Unable to convert test spec: " + stderr)
except Exception as ex:
return error("Unable to convert test spec: " + str(ex))
returned = pscheduler.json_load(stdout)
returned.insert(0, test)
return ok(pscheduler.json_dump(returned))
def tests_name_participants(name):
spec = request.args.get('spec')
if spec is None:
return bad_request("No test spec provided")
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", name,
"participants"],
stdin = spec
)
except KeyError:
return bad_request("Invalid spec")
except Exception as ex:
return bad_request(ex)
if returncode != 0:
return bad_request(stderr)
# If this fails because of bad JSON, an exception will be thrown,
# caught and logged.
return json_response(pscheduler.json_load(stdout))
def tests_name_lead(name):
spec = request.args.get('spec')
if spec is None:
return bad_request("No test spec provided")
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test", name,
"participants"],
stdin = spec
)
except KeyError:
return bad_request("Invalid spec")
except Exception as ex:
return bad_request(ex)
if returncode != 0:
return bad_request(stderr)
part_list = pscheduler.json_load(stdout)
lead = part_list['participants'][0]
return json_response(lead)
def tasks_uuid(uuid):
if request.method == 'GET':
# Get a task, adding server-derived details if a 'detail'
# argument is present.
try:
cursor = dbcursor_query("""
SELECT
task.json,
task.added,
task.start,
task.slip,
task.duration,
task.post,
task.runs,
task.participants,
scheduling_class.anytime,
scheduling_class.exclusive,
scheduling_class.multi_result,
task.participant,
task.enabled,
task.cli
FROM
task
JOIN test ON test.id = task.test
JOIN scheduling_class
ON scheduling_class.id = test.scheduling_class
WHERE uuid = %s
""", [uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if row is None:
return not_found()
json = row[0]
# Redact anything in the test spec or archivers that's marked
# private as well as _key at the top level if there is one.
if "_key" in json:
json["_key"] = None
json["test"]["spec"] = pscheduler.json_decomment(
json["test"]["spec"], prefix="_", null=True)
try:
for archive in range(0,len(json["archives"])):
json["archives"][archive]["data"] = pscheduler.json_decomment(
json["archives"][archive]["data"], prefix="_", null=True)
except KeyError:
pass # Don't care if not there.
# Add details if we were asked for them.
if arg_boolean('detail'):
part_list = row[7];
# The database is not supposed to allow this, but spit out
# a sane default as a last resort in case it happens.
if part_list is None:
part_list = [None]
if row[10] == 0 and part_list[0] is None:
part_list[0] = pscheduler.api_this_host()
json['detail'] = {
'added': None if row[1] is None \
else pscheduler.datetime_as_iso8601(row[1]),
'start': None if row[2] is None \
else pscheduler.datetime_as_iso8601(row[2]),
'slip': None if row[3] is None \
else pscheduler.timedelta_as_iso8601(row[3]),
'duration': None if row[4] is None \
else pscheduler.timedelta_as_iso8601(row[4]),
'post': None if row[5] is None \
else pscheduler.timedelta_as_iso8601(row[5]),
'runs': None if row[6] is None \
else int(row[6]),
'participants': part_list,
'anytime': row[8],
'exclusive': row[9],
'multi-result': row[10],
'enabled': row[12],
'cli': row[13]
}
return ok_json(json)
elif request.method == 'POST':
log.debug("Posting to %s", uuid)
log.debug("Data is %s", request.data)
# TODO: This is only for participant 1+
# TODO: This should probably a PUT and not a POST.
try:
json_in = pscheduler.json_load(request.data)
except ValueError:
return bad_request("Invalid JSON")
log.debug("JSON is %s", json_in)
try:
participant = arg_cardinal('participant')
except ValueError as ex:
return bad_request("Invalid participant: " + str(ex))
log.debug("Participant %d", participant)
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on task")
processor, whynot = limitprocessor()
if processor is None:
message = "Limit processor is not initialized: %s" % whynot
log.debug(message)
return no_can_do(message)
# TODO: This is cooked up in two places. Make a function of it.
hints = {
"ip": request.remote_addr
}
hints_data = pscheduler.json_dump(hints)
passed, diags = processor.process(json_in["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
log.debug("Limits passed")
# TODO: Pluck UUID from URI
uuid = url_last_in_path(request.url)
log.debug("Posting task %s", uuid)
try:
cursor = dbcursor_query(
"SELECT * FROM api_task_post(%s, %s, %s, %s)",
[request.data, hints_data, participant, uuid])
except Exception as ex:
return error(str(ex))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
# TODO: Assert that rowcount is 1
log.debug("All done: %s", base_url())
return ok(base_url())
elif request.method == 'DELETE':
parsed = list(urlparse.urlsplit(request.url))
parsed[1] = "%s"
template = urlparse.urlunsplit(parsed)
try:
cursor = dbcursor_query(
"SELECT api_task_disable(%s, %s)", [uuid, template])
except Exception as ex:
return error(str(ex))
return ok()
else:
return not_allowed()
def tasks():
if request.method == 'GET':
expanded = is_expanded()
query = """
SELECT json, uuid
FROM task
"""
args = []
try:
json_query = arg_json("json")
except ValueError as ex:
return bad_request(str(ex))
if json_query is not None:
query += "WHERE json @> %s"
args.append(request.args.get("json"))
query += " ORDER BY added"
try:
cursor = dbcursor_query(query, args)
except Exception as ex:
return error(str(ex))
result = []
for row in cursor:
url = base_url(row[1])
if not expanded:
result.append(url)
continue
row[0]['href'] = url
result.append(row[0])
return json_response(result)
elif request.method == 'POST':
try:
task = pscheduler.json_load(request.data)
except ValueError:
return bad_request("Invalid JSON:" + request.data)
# TODO: Validate the JSON against a TaskSpecification
# See if the task spec is valid
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
task['test']['type'], "spec-is-valid" ],
stdin = pscheduler.json_dump(task['test']['spec'])
)
if returncode != 0:
return error("Invalid test specification: " + stderr)
except Exception as ex:
return error("Unable to validate test spec: " + str(ex))
log.debug("Validated test: %s", pscheduler.json_dump(task['test']))
# Find the participants
try:
returncode, stdout, stderr = pscheduler.run_program(
[ "pscheduler", "internal", "invoke", "test",
task['test']['type'], "participants" ],
stdin = pscheduler.json_dump(task['test']['spec'])
)
if returncode != 0:
return error("Unable to determine participants: " + stderr)
participants = [ host if host is not None
else pscheduler.api_this_host()
for host in pscheduler.json_load(stdout)["participants"] ]
except Exception as ex:
return error("Unable to determine participants: " + str(ex))
nparticipants = len(participants)
# TODO: The participants must be unique. This should be
# verified by fetching the host name from each one.
#
# TOOL SELECTION
#
# TODO: Need to provide for tool being specified by the task
# package.
tools = []
for participant in participants:
try:
# TODO: This will fail with a very large test spec.
status, result = pscheduler.url_get(
pscheduler.api_url(participant, "tools"),
params={ 'test': pscheduler.json_dump(task['test']) }
)
if status != 200:
raise Exception("%d: %s" % (status, result))
tools.append(result)
except Exception as ex:
return error("Error getting tools from %s: %s" \
% (participant, str(ex)))
log.debug("Participant %s offers tools %s", participant, tools)
if len(tools) != nparticipants:
return error("Didn't get a full set of tool responses")
if "tools" in task:
tool = pick_tool(tools, pick_from=task['tools'])
else:
tool = pick_tool(tools)
if tool is None:
# TODO: This could stand some additional diagnostics.
return no_can_do("Couldn't find a tool in common among the participants.")
task['tool'] = tool
#
# TASK CREATION
#
task_data = pscheduler.json_dump(task)
log.debug("Task data: %s", task_data)
tasks_posted = []
# Evaluate the task against the limits and reject the request
# if it doesn't pass.
log.debug("Checking limits on %s", task["test"])
(processor, whynot) = limitprocessor()
if processor is None:
log.debug("Limit processor is not initialized. %s", whynot)
return no_can_do("Limit processor is not initialized: %s" % whynot)
# TODO: This is cooked up in two places. Make a function of it.
hints = {
"ip": request.remote_addr
}
hints_data = pscheduler.json_dump(hints)
log.debug("Processor = %s" % processor)
passed, diags = processor.process(task["test"], hints)
if not passed:
return forbidden("Task forbidden by limits:\n" + diags)
# Post the lead with the local database, which also assigns
# its UUID. Make it disabled so the scheduler doesn't try to
# do anything with it until the task has been submitted to all
# of the other participants.
try:
cursor = dbcursor_query("SELECT * FROM api_task_post(%s, %s, 0, NULL, FALSE)",
[task_data, hints_data], onerow=True)
except Exception as ex:
return error(str(ex.diag.message_primary))
if cursor.rowcount == 0:
return error("Task post failed; poster returned nothing.")
task_uuid = cursor.fetchone()[0]
log.debug("Tasked lead, UUID %s", task_uuid)
# Other participants get the UUID forced upon them.
for participant in range(1,nparticipants):
part_name = participants[participant]
try:
log.debug("Tasking %d@%s: %s", participant, part_name, task_data)
post_url = pscheduler.api_url(part_name,
'tasks/' + task_uuid)
log.debug("Posting task to %s", post_url)
status, result = pscheduler.url_post(
post_url,
params={ 'participant': participant },
data=task_data,
json=False,
throw=False)
log.debug("Remote returned %d: %s", status, result)
if status != 200:
raise Exception("Unable to post task to %s: %s"
% (part_name, result))
tasks_posted.append(result)
except Exception as ex:
log.exception()
for url in tasks_posted:
# TODO: Handle failure?
status, result = requests.delete(url)
try:
dbcursor_query("SELECT api_task_delete(%s)",
[task_uuid])
except Exception as ex:
log.exception()
return error("Error while tasking %d@%s: %s" % (participant, part_name, ex))
# Enable the task so the scheduler will schedule it.
try:
dbcursor_query("SELECT api_task_enable(%s)", [task_uuid])
except Exception as ex:
log.exception()
return error("Failed to enable task %s. See system logs." % task_uuid)
log.debug("Task enabled for scheduling.")
return ok_json("%s/%s" % (request.base_url, task_uuid))
else:
return not_allowed()
def tasks_uuid_runs(task):
if request.method == 'GET':
query = "SELECT '" + base_url() + """/' || run.uuid
FROM
run
JOIN task ON task.id = run.task
WHERE
task.uuid = %s"""
args = [task]
try:
start_time = arg_datetime('start')
if start_time is not None:
query += " AND lower(times) >= %s"
args.append(start_time)
end_time = arg_datetime('end')
if end_time is not None:
query += " AND upper(times) <= %s"
args.append(end_time)
if arg_boolean('upcoming'):
query += " AND (times @> normalized_now() OR lower(times) > normalized_now())"
query += " AND state IN (run_state_pending(), run_state_on_deck(), run_state_running(), run_state_nonstart())"
query += " ORDER BY times"
limit = arg_cardinal('limit')
if limit is not None:
query += " LIMIT " + str(limit)
# TODO: This should be exapandable
except ValueError as ex:
return bad_request(str(ex))
return json_query_simple(query, args, empty_ok=True)
elif request.method == 'POST':
log.debug("Run POST: %s --> %s", request.url, request.data)
try:
data = pscheduler.json_load(request.data)
start_time = pscheduler.iso8601_as_datetime(data['start-time'])
except KeyError:
return bad_request("Missing start time")
except ValueError:
return bad_request("Invalid JSON:" + request.data)
try:
passed, diags = __evaluate_limits(task, start_time)
except Exception as ex:
log.exception()
return error(str(ex))
try:
log.debug("Posting run for task %s starting %s"
% (task, start_time))
cursor = dbcursor_query("SELECT api_run_post(%s, %s, NULL, %s)",
[task, start_time, diags], onerow=True)
uuid = cursor.fetchone()[0]
except Exception as ex:
log.exception()
return error(str(ex))
url = base_url() + '/' + uuid
log.debug("New run posted to %s", url)
return ok_json(url)
else:
return not_allowed()
def tasks_uuid_runs_run(task, run):
if task is None:
return bad_request("Missing or invalid task")
if run is None:
return bad_request("Missing or invalid run")
if request.method == 'GET':
# Wait for there to be a local result
wait_local = arg_boolean('wait-local')
# Wait for there to be a merged result
wait_merged = arg_boolean('wait-merged')
if wait_local and wait_merged:
return error("Cannot wait on local and merged results")
# If asked for 'first', dig up the first run and use its UUID.
if run == 'first':
# 60 tries at 0.5s intervals == 30 sec.
tries = 60
while tries > 0:
try:
run = __runs_first_run(task)
except Exception as ex:
log.exception()
return error(str(ex))
if run is not None:
break
time.sleep(1.0)
tries -= 1
if run is None:
return not_found()
# 60 tries at 0.5s intervals == 30 sec.
tries = 60 if (wait_local or wait_merged) else 1
while tries:
try:
cursor = dbcursor_query(
"""
SELECT
lower(run.times),
upper(run.times),
upper(run.times) - lower(run.times),
task.participant,
task.nparticipants,
task.participants,
run.part_data,
run.part_data_full,
run.result,
run.result_full,
run.result_merged,
run_state.enum,
run_state.display,
run.errors,
run.clock_survey
FROM
run
JOIN task ON task.id = run.task
JOIN run_state ON run_state.id = run.state
WHERE
task.uuid = %s
AND run.uuid = %s""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
return not_found()
row = cursor.fetchone()
if not (wait_local or wait_merged):
break
else:
if (wait_local and row[7] is None) \
or (wait_merged and row[9] is None):
time.sleep(0.5)
tries -= 1
else:
break
# Return a result Whether or not we timed out and let the
# client sort it out.
result = {}
# This strips any query parameters and replaces the last item
# with the run, which might be needed if the 'first' option
# was used.
href_path_parts = urlparse.urlparse(request.url).path.split('/')
href_path_parts[-1] = run
href_path = '/'.join(href_path_parts)
href = urlparse.urljoin( request.url, href_path )
result['href'] = href
result['start-time'] = pscheduler.datetime_as_iso8601(row[0])
result['end-time'] = pscheduler.datetime_as_iso8601(row[1])
result['duration'] = pscheduler.timedelta_as_iso8601(row[2])
participant_num = row[3]
result['participant'] = participant_num
result['participants'] = [
pscheduler.api_this_host()
if participant is None and participant_num == 0
else participant
for participant in row[5]
]
result['participant-data'] = row[6]
result['participant-data-full'] = row[7]
result['result'] = row[8]
result['result-full'] = row[9]
result['result-merged'] = row[10]
result['state'] = row[11]
result['state-display'] = row[12]
result['errors'] = row[13]
if row[14] is not None:
result['clock-survey'] = row[14]
result['task-href'] = root_url('tasks/' + task)
result['result-href'] = href + '/result'
return json_response(result)
elif request.method == 'PUT':
log.debug("Run PUT %s", request.url)
# Get the JSON from the body
try:
run_data = pscheduler.json_load(request.data)
except ValueError:
log.exception()
log.debug("Run data was %s", request.data)
return error("Invalid or missing run data")
# If the run doesn't exist, take the whole thing as if it were
# a POST.
try:
cursor = dbcursor_query(
"SELECT EXISTS (SELECT * FROM run WHERE uuid = %s)",
[run], onerow=True)
except Exception as ex:
log.exception()
return error(str(ex))
if not cursor.fetchone()[0]:
log.debug("Record does not exist; full PUT.")
try:
start_time = \
pscheduler.iso8601_as_datetime(run_data['start-time'])
except KeyError:
return bad_request("Missing start time")
except ValueError:
return bad_request("Invalid start time")
passed, diags = __evaluate_limits(task, start_time)
try:
cursor = dbcursor_query("SELECT api_run_post(%s, %s, %s)",
[task, start_time, run], onerow=True)
log.debug("Full put of %s, got back %s", run, cursor.fetchone()[0])
except Exception as ex:
log.exception()
return error(str(ex))
return ok()
# For anything else, only one thing can be udated at a time,
# and even that is a select subset.
log.debug("Record exists; partial PUT.")
if 'part-data-full' in run_data:
log.debug("Updating part-data-full from %s", run_data)
try:
part_data_full = \
pscheduler.json_dump(run_data['part-data-full'])
except KeyError:
return bad_request("Missing part-data-full")
except ValueError:
return bad_request("Invalid part-data-full")
log.debug("Full data is: %s", part_data_full)
try:
cursor = dbcursor_query("""
UPDATE
run
SET
part_data_full = %s
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""",
[ part_data_full, run, task])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount != 1:
return not_found()
log.debug("Full data updated")
return ok()
elif 'result-full' in run_data:
log.debug("Updating result-full from %s", run_data)
try:
result_full = \
pscheduler.json_dump(run_data['result-full'])
except KeyError:
return bad_request("Missing result-full")
except ValueError:
return bad_request("Invalid result-full")
try:
succeeded = bool(run_data['succeeded'])
except KeyError:
return bad_request("Missing success value")
except ValueError:
return bad_request("Invalid success value")
log.debug("Updating result-full: JSON %s", result_full)
log.debug("Updating result-full: Run %s", run)
log.debug("Updating result-full: Task %s", task)
try:
cursor = dbcursor_query("""
UPDATE
run
SET
result_full = %s,
state = CASE %s
WHEN TRUE THEN run_state_finished()
ELSE run_state_failed()
END
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""",
[ result_full, succeeded, run, task ])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount != 1:
return not_found()
return ok()
elif request.method == 'DELETE':
# TODO: If this is the lead, the run's counterparts on the
# other participating nodes need to be removed as well.
try:
cursor = dbcursor_query("""
DELETE FROM run
WHERE
task in (SELECT id FROM task WHERE uuid = %s)
AND uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
return ok() if cursor.rowcount == 1 else not_found()
else:
return not_allowed()
def tasks_uuid_runs(task):
if not uuid_is_valid(task):
return not_found()
if request.method == 'GET':
query = "SELECT '" + base_url() + """/' || run.uuid
FROM
run
JOIN task ON task.id = run.task
WHERE
task.uuid = %s"""
args = [task]
try:
start_time = arg_datetime('start')
if start_time is not None:
query += " AND lower(times) >= %s"
args.append(start_time)
end_time = arg_datetime('end')
if end_time is not None:
query += " AND upper(times) <= %s"
args.append(end_time)
if arg_boolean('upcoming'):
query += " AND (times @> normalized_now() OR lower(times) > normalized_now())"
query += " AND state IN (run_state_pending(), run_state_on_deck(), run_state_running(), run_state_nonstart())"
query += " ORDER BY times"
limit = arg_cardinal('limit')
if limit is not None:
query += " LIMIT " + str(limit)
# TODO: This should be exapandable
except ValueError as ex:
return bad_request(str(ex))
return json_query_simple(query, args, empty_ok=True)
elif request.method == 'POST':
log.debug("Run POST: %s --> %s", request.url, request.data)
try:
requester, key = task_requester_key(task)
if requester is None:
return not_found()
if not access_write_task(requester, key):
return forbidden()
except Exception as ex:
return error(str(ex))
try:
data = pscheduler.json_load(request.data, max_schema=1)
start_time = pscheduler.iso8601_as_datetime(data['start-time'])
except KeyError:
return bad_request("Missing start time")
except ValueError as ex:
return bad_request("Invalid JSON: %s" % (str(ex)))
try:
passed, diags, response = __evaluate_limits(task, start_time)
except Exception as ex:
log.exception()
return error(str(ex))
if response is not None:
return response
try:
log.debug("Posting run for task %s starting %s" %
(task, start_time))
cursor = dbcursor_query(
"SELECT * FROM api_run_post(%s, %s, NULL, %s)",
[task, start_time, diags],
onerow=True)
succeeded, uuid, conflicts, error_message = cursor.fetchone()
cursor.close()
if conflicts:
return conflict(error_message)
if error_message:
return error(error_message)
except Exception as ex:
log.exception()
return error(str(ex))
url = base_url() + '/' + uuid
log.debug("New run posted to %s", url)
return ok_json(url)
else:
return not_allowed()
def tasks_uuid_runs_run(task, run):
if not uuid_is_valid(task):
return not_found()
if ((request.method in ['PUT', 'DELETE'] and not uuid_is_valid(run))
or (run not in ['first', 'next'] and not uuid_is_valid(run))):
return not_found()
if request.method == 'GET':
# Wait for there to be a local result
wait_local = arg_boolean('wait-local')
# Wait for there to be a merged result
wait_merged = arg_boolean('wait-merged')
if wait_local and wait_merged:
return bad_request("Cannot wait on local and merged results")
# Figure out how long to wait in seconds. Zero means don't
# wait.
wait_time = arg_integer('wait')
if wait_time is None:
wait_time = 30
if wait_time < 0:
return bad_request("Wait time must be >= 0")
# If asked for 'first', dig up the first run and use its UUID.
if run in ['next', 'first']:
future = run == 'next'
wait_interval = 0.5
tries = int(wait_time / wait_interval) if wait_time > 0 \
else 1
while tries > 0:
try:
run = __runs_first_run(task, future)
except Exception as ex:
log.exception()
return error(str(ex))
if run is not None:
break
if wait_time > 0:
time.sleep(1.0)
tries -= 1
if run is None:
return not_found()
# 60 tries at 0.5s intervals == 30 sec.
tries = 60 if (wait_local or wait_merged) else 1
while tries:
try:
cursor = dbcursor_query(
"""
SELECT
lower(run.times),
upper(run.times),
upper(run.times) - lower(run.times),
task.participant,
task.nparticipants,
task.participants,
run.part_data,
run.part_data_full,
run.result,
run.result_full,
run.result_merged,
run_state.enum,
run_state.display,
run.errors,
run.clock_survey,
run.id,
archiving_json(run.id),
run.added,
run_state.finished
FROM
run
JOIN task ON task.id = run.task
JOIN run_state ON run_state.id = run.state
WHERE
task.uuid = %s
AND run.uuid = %s""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
if cursor.rowcount == 0:
cursor.close()
return not_found()
row = cursor.fetchone()
cursor.close()
if not (wait_local or wait_merged):
break
else:
if (wait_local and row[8] is None) \
or (wait_merged \
and ( (row[9] is None) or (not row[18]) ) ):
log.debug("Waiting for merged: %s %s", row[9], row[18])
time.sleep(0.5)
tries -= 1
else:
break
# Return a result Whether or not we timed out and let the
# client sort it out.
result = {}
# This strips any query parameters and replaces the last item
# with the run, which might be needed if the 'first' option
# was used.
href_path_parts = urlparse.urlparse(request.url).path.split('/')
href_path_parts[-1] = run
href_path = '/'.join(href_path_parts)
href = urlparse.urljoin(request.url, href_path)
result['href'] = href
result['start-time'] = pscheduler.datetime_as_iso8601(row[0])
result['end-time'] = pscheduler.datetime_as_iso8601(row[1])
result['duration'] = pscheduler.timedelta_as_iso8601(row[2])
participant_num = row[3]
result['participant'] = participant_num
result['participants'] = [
server_netloc()
if participant is None and participant_num == 0 else participant
for participant in row[5]
]
result['participant-data'] = row[6]
result['participant-data-full'] = row[7]
result['result'] = row[8]
result['result-full'] = row[9]
result['result-merged'] = row[10]
result['state'] = row[11]
result['state-display'] = row[12]
result['errors'] = row[13]
if row[14] is not None:
result['clock-survey'] = row[14]
if row[16] is not None:
result['archivings'] = row[16]
if row[17] is not None:
result['added'] = pscheduler.datetime_as_iso8601(row[17])
result['task-href'] = root_url('tasks/' + task)
result['result-href'] = href + '/result'
return json_response(result)
elif request.method == 'PUT':
log.debug("Run PUT %s", request.url)
try:
requester, key = task_requester_key(task)
if requester is None:
return not_found()
if not access_write_task(requester, key):
return forbidden()
except Exception as ex:
return error(str(ex))
# Get the JSON from the body
try:
run_data = pscheduler.json_load(request.data, max_schema=1)
except ValueError:
log.exception()
log.debug("Run data was %s", request.data)
return bad_request("Invalid or missing run data")
# If the run doesn't exist, take the whole thing as if it were
# a POST.
try:
cursor = dbcursor_query(
"SELECT EXISTS (SELECT * FROM run WHERE uuid = %s)", [run],
onerow=True)
except Exception as ex:
log.exception()
return error(str(ex))
fetched = cursor.fetchone()[0]
cursor.close()
if not fetched:
log.debug("Record does not exist; full PUT.")
try:
start_time = \
pscheduler.iso8601_as_datetime(run_data['start-time'])
except KeyError:
return bad_request("Missing start time")
except ValueError:
return bad_request("Invalid start time")
try:
passed, diags, response = __evaluate_limits(task, start_time)
if response is not None:
return response
cursor = dbcursor_query(
"SELECT * FROM api_run_post(%s, %s, %s)",
[task, start_time, run],
onerow=True)
succeeded, uuid, conflicts, error_message = cursor.fetchone()
cursor.close()
if conflicts:
return conflict(error_message)
if not succeeded:
return error(error_message)
log.debug("Full put of %s, got back %s", run, uuid)
except Exception as ex:
log.exception()
return error(str(ex))
return ok()
# For anything else, only one thing can be udated at a time,
# and even that is a select subset.
log.debug("Record exists; partial PUT.")
if 'part-data-full' in run_data:
log.debug("Updating part-data-full from %s", run_data)
try:
part_data_full = \
pscheduler.json_dump(run_data['part-data-full'])
except KeyError:
return bad_request("Missing part-data-full")
except ValueError:
return bad_request("Invalid part-data-full")
log.debug("Full data is: %s", part_data_full)
try:
cursor = dbcursor_query(
"""
UPDATE
run
SET
part_data_full = %s
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""", [part_data_full, run, task])
except Exception as ex:
log.exception()
return error(str(ex))
rowcount = cursor.rowcount
cursor.close()
if rowcount != 1:
return not_found()
log.debug("Full data updated")
return ok()
elif 'result-full' in run_data:
log.debug("Updating result-full from %s", run_data)
try:
result_full = \
pscheduler.json_dump(run_data['result-full'])
except KeyError:
return bad_request("Missing result-full")
except ValueError:
return bad_request("Invalid result-full")
try:
succeeded = bool(run_data['succeeded'])
except KeyError:
return bad_request("Missing success value")
except ValueError:
return bad_request("Invalid success value")
log.debug("Updating result-full: JSON %s", result_full)
log.debug("Updating result-full: Run %s", run)
log.debug("Updating result-full: Task %s", task)
try:
cursor = dbcursor_query(
"""
UPDATE
run
SET
result_full = %s,
state = CASE %s
WHEN TRUE THEN run_state_finished()
ELSE run_state_failed()
END
WHERE
uuid = %s
AND EXISTS (SELECT * FROM task WHERE UUID = %s)
""", [result_full, succeeded, run, task])
except Exception as ex:
log.exception()
return error(str(ex))
rowcount = cursor.rowcount
cursor.close()
if rowcount != 1:
return not_found()
return ok()
elif request.method == 'DELETE':
# TODO: If this is the lead, the run's counterparts on the
# other participating nodes need to be removed as well.
try:
requester, key = task_requester_key(task)
if requester is None:
return not_found()
if not access_write_task(requester, key):
return forbidden()
except Exception as ex:
return error(str(ex))
try:
cursor = dbcursor_query(
"""
DELETE FROM run
WHERE
task in (SELECT id FROM task WHERE uuid = %s)
AND uuid = %s
""", [task, run])
except Exception as ex:
log.exception()
return error(str(ex))
rowcount = cursor.rowcount
cursor.close()
return ok() if rowcount == 1 else not_found()
else:
return not_allowed()
