def tests_name_spec(name): cursor = dbcursor_query("SELECT EXISTS (SELECT * FROM test" " WHERE available AND name = %s)", [ name ]) exists = cursor.fetchone()[0] cursor.close() if not exists: return not_found() try: args = arg_json('args') except ValueError as ex: return bad_request("JSON passed to 'args': %s " % (str(ex))) status, stdout, stderr = pscheduler.run_program( [ 'pscheduler', 'internal', 'invoke', 'test', name, 'cli-to-spec' ], stdin = pscheduler.json_dump(args), timeout=5 ) if status != 0: return bad_request(stderr) # The extra parse here makes 'pretty' work. returned_json = pscheduler.json_load(stdout) return ok_json(returned_json, sanitize=False)
def __init__( self, data # Data suitable for this class ): valid, message = data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.test = data['test'] self.limit = data['limit'] returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", self.test, "limit-is-valid" ], stdin=pscheduler.json_dump(self.limit), # TODO: Is this reasonable? timeout=5) if returncode != 0: raise RuntimeError("Failed to validate limit: %s" % stderr) result = pscheduler.json_load(stdout, max_schema=1) if not result['valid']: raise ValueError("Invalid limit: %s" % result['message'])
def tests_name_spec_is_valid(name): cursor = dbcursor_query( "SELECT EXISTS" " (SELECT * FROM test WHERE available AND name = %s)", [name]) exists = cursor.fetchone()[0] cursor.close() if not exists: return not_found() spec = request.args.get('spec') if spec is None: return bad_request("No test spec provided") try: returncode, stdout, stderr = pscheduler.run_program( ["pscheduler", "internal", "invoke", "test", name, "spec-is-valid"], stdin=spec) if returncode != 0: return error("Unable to validate test spec: %s" % (stderr)) validate_json = pscheduler.json_load(stdout, max_schema=1) return ok_json(validate_json) except Exception as ex: return error("Unable to validate test spec: %s" % (str(ex)))
def run( self, # These are lifted straight from run_program. line_call=None, # Lambda to call when a line arrives timeout=None, # Seconds timeout_ok=False, # Treat timeouts as not being an error fail_message=None, # Exit with this failure message env=None, # Environment for new process, None=existing env_add=None, # Add hash to existing environment attempts=10): # Max attempts to start the process """ Run the chain. Return semantics are the same as for pscheduler.run_program(): a tuple of status, stdout, stderr. """ # TODO: Is there a more-pythonic way to do this than pasting # in all of the args? result = pscheduler.run_program([self.stages[0]], line_call=line_call, timeout=timeout, timeout_ok=timeout_ok, fail_message=fail_message, env=env, env_add=env_add, attempts=attempts) for remove in self.stages: try: pass # os.unlink(remove) except IOError: pass # This is best effort only. return result
def tests_name_spec(name): try: cursor = dbcursor_query("SELECT EXISTS (SELECT * FROM test WHERE NAME = %s)", [ name ]) except Exception as ex: return error(str(ex)) exists = cursor.fetchone()[0] if not exists: return not_found() try: args = arg_json('args') except ValueError: return error("Invalid JSON passed to 'args'") status, stdout, stderr = pscheduler.run_program( [ 'pscheduler', 'internal', 'invoke', 'test', name, 'cli-to-spec' ], stdin = pscheduler.json_dump(args), short = True, ) if status != 0: return bad_request(stderr) # The extra parse here makes 'pretty' work. returned_json = pscheduler.json_load(stdout) return ok_json(returned_json)
def evaluate(self, run): # The proposed run # Dissent if the test isn't our type if run["type"] != self.test: return {"passed": False, "reasons": ["Test is not '%s'" % self.test]} pass_input = {"spec": run["spec"], "limit": self.limit} returncode, stdout, stderr = pscheduler.run_program( ["pscheduler", "internal", "invoke", "test", self.test, "limit-passes"], stdin=pscheduler.json_dump(pass_input), # TODO: Is this reasonable? timeout=5, ) if returncode != 0: raise RuntimeError("Failed to validate limit: %s" % stderr) check_result = pscheduler.json_load(stdout) passed = check_result["passes"] result = {"passed": passed} if not passed: result["reasons"] = check_result["errors"] return result
def tests_name_participants(name): spec = request.args.get('spec') if spec is None: return bad_request("No test spec provided") # HACK: BWCTLBC -- BEGIN lead_bind = request.args.get("lead-bind") if lead_bind is not None: env_add = {"PSCHEDULER_LEAD_BIND_HACK": lead_bind} else: env_add = None # HACK: BWCTLBC --- END try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", name, "participants"], stdin = spec, env_add=env_add # HACK: BWCTLBC ) except KeyError: return bad_request("Invalid spec") except Exception as ex: return bad_request(ex) if returncode != 0: return bad_request(stderr) # If this fails because of bad JSON, an exception will be thrown, # caught and logged. return json_response(pscheduler.json_load(stdout, max_schema=1))
def evaluate( self, run # The proposed run ): # Dissent if the test isn't our type if run["type"] != self.test: return { "passed": False, "reasons": ["Test is not '%s'" % self.test] } pass_input = {"spec": run["spec"], "limit": self.limit} returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", self.test, "limit-passes" ], stdin=pscheduler.json_dump(pass_input), # TODO: Is this reasonable? timeout=5) if returncode != 0: raise RuntimeError("Failed to validate limit: %s" % stderr) check_result = pscheduler.json_load(stdout, max_schema=1) passed = check_result["passes"] result = {"passed": passed, "limit": self.limit} if not passed: result["reasons"] = check_result["errors"] return result
def contexts_name_data_is_valid(name): try: cursor = dbcursor_query( "SELECT EXISTS" " (SELECT * FROM context WHERE available AND NAME = %s)", [name]) except Exception as ex: return error(str(ex)) exists = cursor.fetchone()[0] cursor.close() if not exists: return not_found() data = request.args.get('data') if data is None: return bad_request("No archive data provided") try: returncode, stdout, stderr = pscheduler.run_program( ["pscheduler", "internal", "invoke", "context", name, "data-is-valid"], stdin=data) if returncode != 0: return error("Unable to validate context data: %s" % (stderr)) validate_json = pscheduler.json_load(stdout, max_schema=1) return ok_json(validate_json) except Exception as ex: return error("Unable to validate context data: %s" % (str(ex)))
def run_cmd(self, input, args=[], expected_status=0, json_out=True, expected_stderr=""): """ Run and verify results. Takes following params: Args: input: a string to be fed via stdin to the program being executed args: optional command-line arguments to be given to the command-run. expected_status: the expected return code of the program. default is 0. json_out: indicates whether output should be expected to be valid JSON expected_stderr: string to match against stderr Returns: stdout as json dict if json_out is True (default), otehrwise a string """ #Run command full_cmd = [self.cmd] + args try: status, stdout, stderr = run_program(full_cmd, stdin=input) print self.cmd except: #print stacktrace for any errors traceback.print_exc() self.fail("unable to run command {0}".format(self.cmd)) #check stdout and stderr print stdout print stderr self.assertEqual(status, expected_status) #status should be 0 if expected_status == 0: self.assertFalse(stderr) #stderr should be empty elif expected_stderr: self.assertEquals(stderr, expected_stderr) if not json_out: return stdout #check for valid JSON try: result_json = json.loads(stdout) except: traceback.print_exc() self.fail("Invalid JSON returned by {0}: {1}".format( self.progname, stdout)) return result_json
def get_output(self, args, check_success=True): args.insert(0, "%s/../cli-to-spec" % self.path) # actually run cli-to-spec with the input code, stdout, stderr = pscheduler.run_program(args) if check_success: # make sure it succeeded self.assertEqual(code, 0) # get json out if code != 0: return stderr return json.loads(stdout)
def get_output(self, limit, spec, check_success=True): args = {"limit": limit, "spec": spec} # actually run cli-to-spec with the input code, stdout, stderr = pscheduler.run_program("%s/../limit-passes" % self.path, stdin=json.dumps(args)) if check_success: # make sure it succeeded self.assertEqual(code, 0) # get json out if code != 0: return stderr return json.loads(stdout)
def _run_cmd(self, input, expected_status=0, expected_valid=True, expected_errors=[], match_all_errors=True): #Run command self._init_cmd() try: status, stdout, stderr = run_program([self.vaildate_cmd], stdin=input) except: #print stacktrace for any errors traceback.print_exc() self.fail("unable to run limit-passes command {0}".format( self.vaildate_cmd)) #check stdout and stderr print stdout print stderr self.assertEqual(status, expected_status) #status should be 0 self.assertFalse(stderr) #stderr should be empty #check for valid JSON try: result_json = json.loads(stdout) except: traceback.print_exc() self.fail( "Invalid JSON returned by limit-passes: {0}".format(stdout)) #check fields assert ('passes' in result_json) self.assertEqual(result_json['passes'], expected_valid) if expected_valid: assert ('errors' not in result_json) else: assert ('errors' in result_json) if len(expected_errors) > 0 and match_all_errors: #verify list of errors same length self.assertEqual(len(result_json['errors']), len(expected_errors)) for expected_error in expected_errors: #verify messages are in list assert (expected_error in result_json['errors'])
def tasks_uuid_cli(uuid): if not uuid_is_valid(uuid): return not_found() # Get a task, adding server-derived details if a 'detail' # argument is present. try: cursor = dbcursor_query( """SELECT task.json #>> '{test, spec}', test.name FROM task JOIN test on test.id = task.test WHERE task.uuid = %s""", [uuid]) except Exception as ex: return error(str(ex)) if cursor.rowcount == 0: return not_found() row = cursor.fetchone() if row is None: return not_found() json, test = row try: returncode, stdout, stderr = pscheduler.run_program( ["pscheduler", "internal", "invoke", "test", test, "spec-to-cli"], stdin=json) if returncode != 0: return error("Unable to convert test spec: " + stderr) except Exception as ex: return error("Unable to convert test spec: " + str(ex)) returned = pscheduler.json_load(stdout) returned.insert(0, test) return ok(pscheduler.json_dump(returned))
def __init__(self, data): # Data suitable for this class valid, message = test_data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.test = data["test"] self.limit = data["limit"] returncode, stdout, stderr = pscheduler.run_program( ["pscheduler", "internal", "invoke", "test", self.test, "limit-is-valid"], stdin=pscheduler.json_dump(self.limit), # TODO: Is this reasonable? timeout=5, ) if returncode != 0: raise RuntimeError("Failed to validate limit: %s" % stderr) result = pscheduler.json_load(stdout) if not result["valid"]: raise ValueError("Invalid limit: %s" % result["message"])
def tasks_uuid_cli(uuid): # Get a task, adding server-derived details if a 'detail' # argument is present. try: cursor = dbcursor_query( """SELECT task.json #>> '{test, spec}', test.name FROM task JOIN test on test.id = task.test WHERE task.uuid = %s""", [uuid]) except Exception as ex: return error(str(ex)) if cursor.rowcount == 0: return not_found() row = cursor.fetchone() if row is None: return not_found() json, test = row try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", test, "spec-to-cli" ], stdin = json ) if returncode != 0: return error("Unable to convert test spec: " + stderr) except Exception as ex: return error("Unable to convert test spec: " + str(ex)) returned = pscheduler.json_load(stdout) returned.insert(0, test) return ok(pscheduler.json_dump(returned))
def tests_name_participants(name): spec = request.args.get('spec') if spec is None: return bad_request("No test spec provided") try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", name, "participants"], stdin = spec ) except KeyError: return bad_request("Invalid spec") except Exception as ex: return bad_request(ex) if returncode != 0: return bad_request(stderr) # If this fails because of bad JSON, an exception will be thrown, # caught and logged. return json_response(pscheduler.json_load(stdout))
def tests_name_participants(name): spec = request.args.get('spec') if spec is None: return bad_request("No test spec provided") try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", name, "participants"], stdin = spec, ) except KeyError: return bad_request("Invalid spec") except Exception as ex: return bad_request(ex) if returncode != 0: return bad_request(stderr) # If this fails because of bad JSON, an exception will be thrown, # caught and logged. return json_response(pscheduler.json_load(stdout, max_schema=1))
def tests_name_lead(name): spec = request.args.get('spec') if spec is None: return bad_request("No test spec provided") try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", name, "participants"], stdin = spec ) except KeyError: return bad_request("Invalid spec") except Exception as ex: return bad_request(ex) if returncode != 0: return bad_request(stderr) part_list = pscheduler.json_load(stdout) lead = part_list['participants'][0] return json_response(lead)
def _run_cmd(self, input, expected_status=0, expected_valid=True, expected_error=""): #Run command try: status, stdout, stderr = run_program([self.vaildate_cmd], stdin=input) except: #print stacktrace for any errors traceback.print_exc() self.fail("unable to run limit-is-valid command {0}".format( self.vaildate_cmd)) #check stdout and stderr print stdout print stderr self.assertEqual(status, expected_status) #status should be 0 self.assertFalse(stderr) #stderr should be empty #check for valid JSON try: result_json = json.loads(stdout) except: traceback.print_exc() self.fail( "Invalid JSON returned by limit-is-valid: {0}".format(stdout)) #check fields assert (self.result_valid_field in result_json) self.assertEqual(result_json[self.result_valid_field], expected_valid) if expected_valid: assert (self.error_field not in result_json) else: assert (self.error_field in result_json) if expected_error: self.assertEqual(result_json[self.error_field], expected_error)
def tasks_uuid_runs_run_result(task, run): if not uuid_is_valid(task) or not uuid_is_valid(run): return not_found() wait = arg_boolean('wait') format = request.args.get('format') if format is None: format = 'application/json' if format not in ['application/json', 'text/html', 'text/plain']: return bad_request("Unsupported format " + format) # If asked for 'first', dig up the first run and use its UUID. # This is more for debug convenience than anything else. if run == 'first': try: run = __runs_first_run(task) except Exception as ex: log.exception() return error(str(ex)) if run is None: return not_found() # # Camp on the run for a result # # 40 tries at 0.25s intervals == 10 sec. tries = 40 if wait else 1 while tries: try: cursor = dbcursor_query( """ SELECT test.name, run.result_merged, task.json #> '{test, spec}' FROM run JOIN task ON task.id = run.task JOIN test ON test.id = task.test WHERE task.uuid = %s AND run.uuid = %s """, [task, run]) except Exception as ex: log.exception() return error(str(ex)) if cursor.rowcount == 0: cursor.close() return not_found() # TODO: Make sure we got back one row with two columns. row = cursor.fetchone() cursor.close() if not wait and row[1] is None: time.sleep(0.25) tries -= 1 else: break if tries == 0: return not_found() test_type, merged_result, test_spec = row # JSON requires no formatting. if format == 'application/json': return ok_json(merged_result) if not merged_result['succeeded']: if format == 'text/plain': return ok("Run failed.", mimetype=format) elif format == 'text/html': return ok("<p>Run failed.</p>", mimetype=format) return bad_request("Unsupported format " + format) formatter_input = {"spec": test_spec, "result": merged_result} returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", test_type, "result-format", format ], stdin=pscheduler.json_dump(formatter_input)) if returncode != 0: return error("Failed to format result: " + stderr) return ok(stdout.rstrip(), mimetype=format)
def tasks(): if request.method == 'GET': expanded = is_expanded() query = """ SELECT json, uuid FROM task """ args = [] try: json_query = arg_json("json") except ValueError as ex: return bad_request(str(ex)) if json_query is not None: query += "WHERE json @> %s" args.append(request.args.get("json")) query += " ORDER BY added" try: cursor = dbcursor_query(query, args) except Exception as ex: return error(str(ex)) result = [] for row in cursor: url = base_url(row[1]) if not expanded: result.append(url) continue row[0]['href'] = url result.append(row[0]) return json_response(result) elif request.method == 'POST': try: task = pscheduler.json_load(request.data) except ValueError: return bad_request("Invalid JSON:" + request.data) # TODO: Validate the JSON against a TaskSpecification # See if the task spec is valid try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", task['test']['type'], "spec-is-valid" ], stdin = pscheduler.json_dump(task['test']['spec']) ) if returncode != 0: return error("Invalid test specification: " + stderr) except Exception as ex: return error("Unable to validate test spec: " + str(ex)) log.debug("Validated test: %s", pscheduler.json_dump(task['test'])) # Find the participants try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", task['test']['type'], "participants" ], stdin = pscheduler.json_dump(task['test']['spec']) ) if returncode != 0: return error("Unable to determine participants: " + stderr) participants = [ host if host is not None else pscheduler.api_this_host() for host in pscheduler.json_load(stdout)["participants"] ] except Exception as ex: return error("Unable to determine participants: " + str(ex)) nparticipants = len(participants) # TODO: The participants must be unique. This should be # verified by fetching the host name from each one. # # TOOL SELECTION # # TODO: Need to provide for tool being specified by the task # package. tools = [] for participant in participants: try: # TODO: This will fail with a very large test spec. status, result = pscheduler.url_get( pscheduler.api_url(participant, "tools"), params={ 'test': pscheduler.json_dump(task['test']) } ) if status != 200: raise Exception("%d: %s" % (status, result)) tools.append(result) except Exception as ex: return error("Error getting tools from %s: %s" \ % (participant, str(ex))) log.debug("Participant %s offers tools %s", participant, tools) if len(tools) != nparticipants: return error("Didn't get a full set of tool responses") if "tools" in task: tool = pick_tool(tools, pick_from=task['tools']) else: tool = pick_tool(tools) if tool is None: # TODO: This could stand some additional diagnostics. return no_can_do("Couldn't find a tool in common among the participants.") task['tool'] = tool # # TASK CREATION # task_data = pscheduler.json_dump(task) log.debug("Task data: %s", task_data) tasks_posted = [] # Evaluate the task against the limits and reject the request # if it doesn't pass. log.debug("Checking limits on %s", task["test"]) (processor, whynot) = limitprocessor() if processor is None: log.debug("Limit processor is not initialized. %s", whynot) return no_can_do("Limit processor is not initialized: %s" % whynot) # TODO: This is cooked up in two places. Make a function of it. hints = { "ip": request.remote_addr } hints_data = pscheduler.json_dump(hints) log.debug("Processor = %s" % processor) passed, diags = processor.process(task["test"], hints) if not passed: return forbidden("Task forbidden by limits:\n" + diags) # Post the lead with the local database, which also assigns # its UUID. Make it disabled so the scheduler doesn't try to # do anything with it until the task has been submitted to all # of the other participants. try: cursor = dbcursor_query("SELECT * FROM api_task_post(%s, %s, 0, NULL, FALSE)", [task_data, hints_data], onerow=True) except Exception as ex: return error(str(ex.diag.message_primary)) if cursor.rowcount == 0: return error("Task post failed; poster returned nothing.") task_uuid = cursor.fetchone()[0] log.debug("Tasked lead, UUID %s", task_uuid) # Other participants get the UUID forced upon them. for participant in range(1,nparticipants): part_name = participants[participant] try: log.debug("Tasking %d@%s: %s", participant, part_name, task_data) post_url = pscheduler.api_url(part_name, 'tasks/' + task_uuid) log.debug("Posting task to %s", post_url) status, result = pscheduler.url_post( post_url, params={ 'participant': participant }, data=task_data, json=False, throw=False) log.debug("Remote returned %d: %s", status, result) if status != 200: raise Exception("Unable to post task to %s: %s" % (part_name, result)) tasks_posted.append(result) except Exception as ex: log.exception() for url in tasks_posted: # TODO: Handle failure? status, result = requests.delete(url) try: dbcursor_query("SELECT api_task_delete(%s)", [task_uuid]) except Exception as ex: log.exception() return error("Error while tasking %d@%s: %s" % (participant, part_name, ex)) # Enable the task so the scheduler will schedule it. try: dbcursor_query("SELECT api_task_enable(%s)", [task_uuid]) except Exception as ex: log.exception() return error("Failed to enable task %s. See system logs." % task_uuid) log.debug("Task enabled for scheduling.") return ok_json("%s/%s" % (request.base_url, task_uuid)) else: return not_allowed()
def tasks(): if request.method == 'GET': where_clause = "TRUE" args = [] try: json_query = arg_json("json") except ValueError as ex: return bad_request(str(ex)) if json_query is not None: where_clause += " AND task.json @> %s" args.append(request.args.get("json")) where_clause += " ORDER BY added" try: tasks = __tasks_get_filtered(request.base_url, where_clause=where_clause, args=args, expanded=is_expanded(), detail=arg_boolean("detail"), single=False) except Exception as ex: return error(str(ex)) return ok_json(tasks) elif request.method == 'POST': try: task = pscheduler.json_load(request.data, max_schema=1) except ValueError as ex: return bad_request("Invalid task specification: %s" % (str(ex))) # Validate the JSON against a TaskSpecification # TODO: Figure out how to do this without the intermediate object valid, message = pscheduler.json_validate({"": task}, { "type": "object", "properties": { "": { "$ref": "#/pScheduler/TaskSpecification" } }, "required": [""] }) if not valid: return bad_request("Invalid task specification: %s" % (message)) # See if the test spec is valid try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", task['test']['type'], "spec-is-valid" ], stdin=pscheduler.json_dump(task['test']['spec'])) if returncode != 0: return error("Unable to validate test spec: %s" % (stderr)) validate_json = pscheduler.json_load(stdout, max_schema=1) if not validate_json["valid"]: return bad_request( "Invalid test specification: %s" % (validate_json.get("error", "Unspecified error"))) except Exception as ex: return error("Unable to validate test spec: " + str(ex)) log.debug("Validated test: %s", pscheduler.json_dump(task['test'])) # Reject tasks that have archive specs that use transforms. # See ticket #330. try: for archive in task['archives']: if "transform" in archive: return bad_request( "Use of transforms in archives is not yet supported.") except KeyError: pass # Not there # Find the participants try: # HACK: BWCTLBC if "lead-bind" in task: lead_bind_env = { "PSCHEDULER_LEAD_BIND_HACK": task["lead-bind"] } else: lead_bind_env = None returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", task['test']['type'], "participants" ], stdin=pscheduler.json_dump(task['test']['spec']), timeout=5, env_add=lead_bind_env) if returncode != 0: return error("Unable to determine participants: " + stderr) participants = [ host if host is not None else server_netloc() for host in pscheduler.json_load(stdout, max_schema=1)["participants"] ] except Exception as ex: return error("Exception while determining participants: " + str(ex)) nparticipants = len(participants) # TODO: The participants must be unique. This should be # verified by fetching the host name from each one. # # TOOL SELECTION # lead_bind = task.get("lead-bind", None) # TODO: Need to provide for tool being specified by the task # package. tools = [] tool_params = {"test": pscheduler.json_dump(task["test"])} # HACK: BWCTLBC if lead_bind is not None: log.debug("Using lead bind of %s" % str(lead_bind)) tool_params["lead-bind"] = lead_bind for participant_no in range(0, len(participants)): participant = participants[participant_no] try: # Make sure the other participants are running pScheduler participant_api = pscheduler.api_url_hostport(participant) log.debug("Pinging %s" % (participant)) status, result = pscheduler.url_get(participant_api, throw=False, timeout=10, bind=lead_bind) if status == 400: raise TaskPostingException(result) elif status in [ 202, 204, 205, 206, 207, 208, 226, 300, 301, 302, 303, 304, 205, 306, 307, 308 ] \ or ( (status >= 400) and (status <=499) ): raise TaskPostingException( "Host is not running pScheduler") elif status != 200: raise TaskPostingException("returned status %d: %s" % (status, result)) # TODO: This will fail with a very large test spec. status, result = pscheduler.url_get("%s/tools" % (participant_api), params=tool_params, throw=False, bind=lead_bind) if status != 200: raise TaskPostingException("%d: %s" % (status, result)) tools.append(result) except TaskPostingException as ex: return error("Error getting tools from %s: %s" \ % (participant, str(ex))) log.debug("Participant %s offers tools %s", participant, result) if len(tools) != nparticipants: return error("Didn't get a full set of tool responses") if "tools" in task: tool = pick_tool(tools, pick_from=task['tools']) else: tool = pick_tool(tools) if tool is None: # TODO: This could stand some additional diagnostics. return no_can_do( "Couldn't find a tool in common among the participants.") task['tool'] = tool # # TASK CREATION # tasks_posted = [] # Evaluate the task against the limits and reject the request # if it doesn't pass. log.debug("Checking limits on %s", task["test"]) (processor, whynot) = limitprocessor() if processor is None: log.debug("Limit processor is not initialized. %s", whynot) return no_can_do("Limit processor is not initialized: %s" % whynot) hints = request_hints() hints_data = pscheduler.json_dump(hints) log.debug("Processor = %s" % processor) passed, limits_passed, diags = processor.process(task["test"], hints) if not passed: return forbidden("Task forbidden by limits:\n" + diags) # Post the lead with the local database, which also assigns # its UUID. Make it disabled so the scheduler doesn't try to # do anything with it until the task has been submitted to all # of the other participants. try: cursor = dbcursor_query( "SELECT * FROM api_task_post(%s, %s, %s, %s, 0, NULL, FALSE)", [ pscheduler.json_dump(task), participants, hints_data, pscheduler.json_dump(limits_passed) ], onerow=True) except Exception as ex: return error(str(ex.diag.message_primary)) if cursor.rowcount == 0: return error("Task post failed; poster returned nothing.") task_uuid = cursor.fetchone()[0] log.debug("Tasked lead, UUID %s", task_uuid) # Other participants get the UUID and participant list forced upon them. task["participants"] = participants task_data = pscheduler.json_dump(task) for participant in range(1, nparticipants): part_name = participants[participant] log.debug("Tasking participant %s", part_name) try: # Post the task log.debug("Tasking %d@%s: %s", participant, part_name, task_data) post_url = pscheduler.api_url_hostport(part_name, 'tasks/' + task_uuid) log.debug("Posting task to %s", post_url) status, result = pscheduler.url_post( post_url, params={'participant': participant}, data=task_data, bind=lead_bind, json=False, throw=False) log.debug("Remote returned %d: %s", status, result) if status != 200: raise TaskPostingException( "Unable to post task to %s: %s" % (part_name, result)) tasks_posted.append(result) # Fetch the task's details and add the list of limits # passed to our own. status, result = pscheduler.url_get(post_url, params={"detail": True}, bind=lead_bind, throw=False) if status != 200: raise TaskPostingException( "Unable to fetch posted task from %s: %s" % (part_name, result)) log.debug("Fetched %s", result) try: details = result["detail"]["spec-limits-passed"] log.debug("Details from %s: %s", post_url, details) limits_passed.extend(details) except KeyError: pass except TaskPostingException as ex: # Disable the task locally and let it get rid of the # other participants. posted_to = "%s/%s" % (request.url, task_uuid) parsed = list(urlparse.urlsplit(posted_to)) parsed[1] = "%s" template = urlparse.urlunsplit(parsed) try: dbcursor_query("SELECT api_task_disable(%s, %s)", [task_uuid, template]) except Exception: log.exception() return error("Error while tasking %s: %s" % (part_name, ex)) # Update the list of limits passed in the local database # TODO: How do the other participants know about this? log.debug("Limits passed: %s", limits_passed) try: cursor = dbcursor_query( "UPDATE task SET limits_passed = %s::JSON WHERE uuid = %s", [pscheduler.json_dump(limits_passed), task_uuid]) except Exception as ex: return error(str(ex.diag.message_primary)) # Enable the task so the scheduler will schedule it. try: dbcursor_query("SELECT api_task_enable(%s)", [task_uuid]) except Exception as ex: log.exception() return error("Failed to enable task %s. See system logs." % task_uuid) log.debug("Task enabled for scheduling.") return ok_json("%s/%s" % (request.base_url, task_uuid)) else: return not_allowed()
def tasks(): if request.method == 'GET': where_clause = "TRUE" args = [] try: json_query = arg_json("json") except ValueError as ex: return bad_request(str(ex)) if json_query is not None: where_clause += " AND task.json_detail @> %s" args.append(request.args.get("json")) where_clause += " ORDER BY added" tasks = __tasks_get_filtered(request.base_url, where_clause=where_clause, args=args, expanded=is_expanded(), detail=arg_boolean("detail"), single=False) return ok_json(tasks) elif request.method == 'POST': try: task = pscheduler.json_load(request.data, max_schema=3) except ValueError as ex: return bad_request("Invalid task specification: %s" % (str(ex))) # Validate the JSON against a TaskSpecification # TODO: Figure out how to do this without the intermediate object valid, message = pscheduler.json_validate({"": task}, { "type": "object", "properties": { "": { "$ref": "#/pScheduler/TaskSpecification" } }, "required": [""] }) if not valid: return bad_request("Invalid task specification: %s" % (message)) # See if the test spec is valid try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", task['test']['type'], "spec-is-valid" ], stdin=pscheduler.json_dump(task['test']['spec'])) if returncode != 0: return error("Unable to validate test spec: %s" % (stderr)) validate_json = pscheduler.json_load(stdout, max_schema=1) if not validate_json["valid"]: return bad_request( "Invalid test specification: %s" % (validate_json.get("error", "Unspecified error"))) except Exception as ex: return error("Unable to validate test spec: " + str(ex)) log.debug("Validated test: %s", pscheduler.json_dump(task['test'])) # Validate the archives for archive in task.get("archives", []): # Data try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "archiver", archive["archiver"], "data-is-valid" ], stdin=pscheduler.json_dump(archive["data"]), ) if returncode != 0: return error("Unable to validate archive spec: %s" % (stderr)) except Exception as ex: return error("Unable to validate test spec: " + str(ex)) try: returned_json = pscheduler.json_load(stdout) if not returned_json["valid"]: return bad_request("Invalid archiver data: %s" % (returned_json["error"])) except Exception as ex: return error("Internal probelm validating archiver data: %s" % (str(ex))) # Transform, if there was one. if "transform" in archive: transform = archive["transform"] try: _ = pscheduler.JQFilter(filter_spec=transform["script"], args=transform.get("args", {})) except ValueError as ex: return error("Invalid transform: %s" % (str(ex))) # Validate the lead binding if there was one. lead_bind = task.get("lead-bind", None) if lead_bind is not None \ and (pscheduler.address_interface(lead_bind) is None): return bad_request("Lead bind '%s' is not on this host" % (lead_bind)) # Evaluate the task against the limits and reject the request # if it doesn't pass. We do this early so anything else in # the process gets any rewrites. log.debug("Checking limits on %s", task) (processor, whynot) = limitprocessor() if processor is None: log.debug("Limit processor is not initialized. %s", whynot) return no_can_do("Limit processor is not initialized: %s" % whynot) hints, error_response = request_hints() if hints is None: log.debug("Can't come up with valid hints for lead task limits.") return error_response hints_data = pscheduler.json_dump(hints) log.debug("Processor = %s" % processor) passed, limits_passed, diags, new_task, _priority \ = processor.process(task, hints) if not passed: return forbidden("Task forbidden by limits:\n" + diags) if new_task is not None: try: task = new_task returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", task['test']['type'], "spec-is-valid" ], stdin=pscheduler.json_dump(task["test"]["spec"])) if returncode != 0: return error( "Failed to validate rewritten test specification: %s" % (stderr)) validate_json = pscheduler.json_load(stdout, max_schema=1) if not validate_json["valid"]: return bad_request( "Rewritten test specification is invalid: %s" % (validate_json.get("error", "Unspecified error"))) except Exception as ex: return error( "Unable to validate rewritten test specification: " + str(ex)) # Find the participants try: returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", task['test']['type'], "participants" ], stdin=pscheduler.json_dump(task['test']['spec']), timeout=5) if returncode != 0: return error("Unable to determine participants: " + stderr) participants = [ host if host is not None else server_netloc() for host in pscheduler.json_load(stdout, max_schema=1)["participants"] ] except Exception as ex: return error("Exception while determining participants: " + str(ex)) nparticipants = len(participants) # TODO: The participants must be unique. This should be # verified by fetching the host name from each one. # # TOOL SELECTION # # TODO: Need to provide for tool being specified by the task # package. tools = [] tool_params = {"test": pscheduler.json_dump(task["test"])} tool_offers = {} for participant_no in range(0, len(participants)): participant = participants[participant_no] try: # Make sure the other participants are running pScheduler participant_api = pscheduler.api_url_hostport(participant) log.debug("Pinging %s" % (participant)) status, result = pscheduler.url_get(participant_api, throw=False, timeout=10, bind=lead_bind) if status == 400: raise TaskPostingException(result) elif status in [ 202, 204, 205, 206, 207, 208, 226, 300, 301, 302, 303, 304, 205, 306, 307, 308 ] \ or ( (status >= 400) and (status <=499) ): raise TaskPostingException( "Host is not running pScheduler") elif status != 200: raise TaskPostingException("returned status %d: %s" % (status, result)) # TODO: This will fail with a very large test spec. status, result = pscheduler.url_get("%s/tools" % (participant_api), params=tool_params, throw=False, bind=lead_bind) if status != 200: raise TaskPostingException("%d: %s" % (status, result)) tools.append(result) except TaskPostingException as ex: return error("Error getting tools from %s: %s" \ % (participant, str(ex))) log.debug("Participant %s offers tools %s", participant, result) tool_offers[participant] = result if len(tools) != nparticipants: return error("Didn't get a full set of tool responses") if "tools" in task: tool = pick_tool(tools, pick_from=task['tools']) else: tool = pick_tool(tools) # Complain if no usable tool was found if tool is None: offers = [] for participant in participants: participant_offers = tool_offers.get(participant, [{ "name": "nothing" }]) if participant_offers is not None: offer_set = [offer["name"] for offer in participant_offers] else: offer_set = ["nothing"] offers.append("%s offered %s" % (participant, ", ".join(offer_set))) return no_can_do("No tool in common among the participants: %s." % ("; ".join(offers))) task['tool'] = tool # # TASK CREATION # tasks_posted = [] # Post the lead with the local database, which also assigns # its UUID. Make it disabled so the scheduler doesn't try to # do anything with it until the task has been submitted to all # of the other participants. cursor = dbcursor_query( "SELECT * FROM api_task_post(%s, %s, %s, %s, 0, %s, NULL, FALSE, %s)", [ pscheduler.json_dump(task), participants, hints_data, pscheduler.json_dump(limits_passed), task.get("priority", None), diags ], onerow=True) if cursor.rowcount == 0: return error("Task post failed; poster returned nothing.") task_uuid = cursor.fetchone()[0] log.debug("Tasked lead, UUID %s", task_uuid) # Other participants get the UUID and participant list forced upon them. task["participants"] = participants task_params = {"key": task["_key"]} if "_key" in task else {} for participant in range(1, nparticipants): part_name = participants[participant] log.debug("Tasking participant %s", part_name) try: # Post the task log.debug("Tasking %d@%s: %s", participant, part_name, task) post_url = pscheduler.api_url_hostport(part_name, 'tasks/' + task_uuid) task_params["participant"] = participant log.debug("Posting task to %s", post_url) status, result = pscheduler.url_post(post_url, params=task_params, data=task, bind=lead_bind, json=False, throw=False) log.debug("Remote returned %d: %s", status, result) if status != 200: raise TaskPostingException( "Unable to post task to %s: %s" % (part_name, result)) tasks_posted.append(result) # Fetch the task's details and add the list of limits # passed to our own. status, result = pscheduler.url_get(post_url, params={"detail": True}, bind=lead_bind, throw=False) if status != 200: raise TaskPostingException( "Unable to fetch posted task from %s: %s" % (part_name, result)) log.debug("Fetched %s", result) try: details = result["detail"]["spec-limits-passed"] log.debug("Details from %s: %s", post_url, details) limits_passed.extend(details) except KeyError: pass except TaskPostingException as ex: # Disable the task locally and let it get rid of the # other participants. posted_to = "%s/%s" % (request.url, task_uuid) parsed = list(urlparse.urlsplit(posted_to)) parsed[1] = "%s" template = urlparse.urlunsplit(parsed) try: dbcursor_query("SELECT api_task_disable(%s, %s)", [task_uuid, template]) except Exception: log.exception() return error("Error while tasking %s: %s" % (part_name, ex)) # Update the list of limits passed in the local database # TODO: How do the other participants know about this? log.debug("Limits passed: %s", limits_passed) cursor = dbcursor_query( "UPDATE task SET limits_passed = %s::JSON WHERE uuid = %s", [pscheduler.json_dump(limits_passed), task_uuid]) # Enable the task so the scheduler will schedule it. try: dbcursor_query("SELECT api_task_enable(%s)", [task_uuid]) except Exception as ex: log.exception() return error("Failed to enable task %s. See system logs." % task_uuid) log.debug("Task enabled for scheduling.") task_url = "%s/%s" % (request.base_url, task_uuid) # Non-expanded gets just the URL if not arg_boolean("expanded"): return ok_json(task_url) # Expanded gets a redirect to GET+expanded params = [] for arg in ["detail", "pretty"]: if arg_boolean(arg): params.append(arg) if params: task_url += "?%s" % ("&".join(params)) return see_other(task_url) else: return not_allowed()
def clock_state(): """ Determine the state of the system clock and return a hash of information conforming to the definition of a SystemClockStatus object as described in the JSON dictionary. time - Current system time as an ISO 8601 string synchronized - Whether or not the clock is synchronized to an outside source. source - The source of synchronization. Currently, the only valid value is "ntp." Not provided if not synchronized. reference - A human-readable string describing the source. Not provided if not synchronized. offset - A float indicating the estimated clock offset. Not provided if not synchronized. error - """ status, stdout, stderr = pscheduler.run_program(["chronyc", " tracking"]) if "FileNotFound" not in stderr: if "506 Cannot talk to daemon" not in stdout: return chrony_clock_state(stdout, stderr) adjtime = ntp_adjtime() system_synchronized = adjtime.synchronized # Format the local time with offset as ISO 8601. Python's # strftime() only does "-0400" format; we need "-04:00". utc = datetime.datetime.utcnow() local_tz = tzlocal.get_localzone() time_here = pytz.utc.localize(utc).astimezone(local_tz) raw_offset = time_here.strftime("%z") if len(raw_offset): offset = raw_offset[:3] + ":" + raw_offset[-2:] else: offset = "" result = { "time": time_here.strftime("%Y-%m-%dT%H:%M:%S.%f") + offset, "synchronized": system_synchronized } if system_synchronized: # Assume NTP for the time being try: ntp = ntplib.NTPClient().request("127.0.0.1") result["offset"] = ntp.offset result["source"] = "ntp" result["reference"] = "%s from %s" % (ntplib.stratum_to_text( ntp.stratum), ntplib.ref_id_to_text(ntp.ref_id)) except Exception as ex: result["synchronized"] = False result["error"] = str(ex) return result
def tasks_uuid_runs_run_result(task, run): if task is None: return bad_request("Missing or invalid task") if run is None: return bad_request("Missing or invalid run") wait = arg_boolean('wait') format = request.args.get('format') if format is None: format = 'application/json' if format not in [ 'application/json', 'text/html', 'text/plain' ]: return bad_request("Unsupported format " + format) # If asked for 'first', dig up the first run and use its UUID. # This is more for debug convenience than anything else. if run == 'first': try: run = __runs_first_run(task) except Exception as ex: log.exception() return error(str(ex)) if run is None: return not_found() # # Camp on the run for a result # # 40 tries at 0.25s intervals == 10 sec. tries = 40 if wait else 1 while tries: try: cursor = dbcursor_query(""" SELECT test.name, run.result_merged, task.json #> '{test, spec}' FROM run JOIN task ON task.id = run.task JOIN test ON test.id = task.test WHERE task.uuid = %s AND run.uuid = %s """, [task, run]) except Exception as ex: log.exception() return error(str(ex)) if cursor.rowcount == 0: return not_found() # TODO: Make sure we got back one row with two columns. row = cursor.fetchone() if not wait and row[1] is None: time.sleep(0.25) tries -= 1 else: break if tries == 0: return not_found() test_type, merged_result, test_spec = row # JSON requires no formatting. if format == 'application/json': return ok_json(merged_result) if not merged_result['succeeded']: if format == 'text/plain': return ok("Test failed.", mimetype=format) elif format == 'text/html': return ok("<p>Test failed.</p>", mimetype=format) return error("Unsupported format " + format) formatter_input = { "spec": test_spec, "result": merged_result } returncode, stdout, stderr = pscheduler.run_program( [ "pscheduler", "internal", "invoke", "test", test_type, "result-format", format ], stdin = pscheduler.json_dump(formatter_input) ) if returncode != 0: return error("Failed to format result: " + stderr) return ok(stdout.rstrip(), mimetype=format)