def json_load(source=None, exit_on_error=False, strip=True): """ Load a blob of JSON and exit with failure if it didn't read. Arguments: source - String or open file containing the JSON. If not specified, sys.stdin will be used. exit_on_error - Use the pScheduler failure mechanism to complain and exit the program. (Default False) strip - Remove all pairs whose names begin with '#'. This is a low-budget way to support comments wthout requiring a parser that understands them. (Default True) """ if source is None: source = sys.stdin try: if type(source) is str or type(source) is unicode: json_in = loads(str(source)) elif type(source) is file: json_in = load(source) else: raise Exception("Internal error: bad source type ", type(source)) except ValueError as ex: # TODO: Make this consistent and fix scripts that use it. if type(source) is str or not exit_on_error: raise ValueError("Invalid JSON: " + str(ex)) else: pscheduler.fail("Invalid JSON: " + str(ex)) return json_decomment(json_in) if strip else json_in
def load_schema(schema_file): #find schema file schema_path = os.path.dirname(sys.argv[0]) if schema_path and not schema_path.endswith(os.sep): schema_path += os.sep try: schema = json.loads(open(schema_path + schema_file).read()) except Exception, e: pscheduler.fail("Problem loading schema file: %s" % e)
def add_additional_metadata(self, test_spec={}): #this should not happen if not self.test_type: pscheduler.fail("Developer error. The test type must be set if storing a raw record.") for field in test_spec: key = "pscheduler-%s-%s" % (self.test_type, field) val = test_spec[field] self._parse_test_spec_field(key, val)
def handle_storage_error(result, attempts=0, policy=[]): #build object retry = False archive_err_result = { 'succeeded': False, 'error': result } policy_attempt_sum = 0 for p in policy: policy_attempt_sum += p['attempts'] if policy_attempt_sum > attempts: retry = True archive_err_result['retry'] = p['wait'] break if retry: pscheduler.succeed_json(archive_err_result) else: pscheduler.fail("Archiver permanently abandoned registering test after %d attempt(s): %s" % (attempts+1, result))
def stat_control_pause(): cursor = dbcursor_query(""" SELECT control_is_paused(), date_trunc('second', pause_runs_until - now()), pause_runs_until = tstz_infinity() FROM control""") if cursor.rowcount != 1: pscheduler.fail("Got back more data than expected.") (is_paused, left, infinite) = cursor.fetchone() cursor.close() result = {"is_paused": is_paused} if is_paused: result["infinite"] = infinite if not infinite: result["remaining"] = pscheduler.timedelta_as_iso8601(left) return ok_json(result)
def json_load(source=None, exit_on_error=False, strip=True, max_schema=None): """ Load a blob of JSON and exit with failure if it didn't read. Arguments: source - String or open file containing the JSON. If not specified, sys.stdin will be used. exit_on_error - Use the pScheduler failure mechanism to complain and exit the program. (Default False) strip - Remove all pairs whose names begin with '#'. This is a low-budget way to support comments wthout requiring a parser that understands them. (Default True) max_schema - Check for a "schema" of no more than this integer value. """ if source is None: source = sys.stdin try: if type(source) is str or type(source) is unicode: json_in = loads(str(source)) elif type(source) is file: json_in = load(source) else: raise Exception("Internal error: bad source type ", type(source)) except ValueError as ex: # TODO: Make this consistent and fix scripts that use it. if type(source) is str or not exit_on_error: raise ValueError("Invalid JSON: " + str(ex)) else: pscheduler.fail("Invalid JSON: " + str(ex)) if max_schema is not None: json_check_schema(json_in, max_schema) return json_decomment(json_in) if strip else json_in
def run_program( argv, # Program name and args stdin=None, # What to send to stdin line_call=None, # Lambda to call when a line arrives timeout=None, # Seconds timeout_ok=False, # Treat timeouts as not being an error fail_message=None, # Exit with this failure message env=None, # Environment for new process, None=existing env_add=None, # Add hash to existing environment attempts=10): # Max attempts to start the process """ Run a program and return the results. Arguments: argv - Array containing arguments, including name of program stdin=s - String containing what should be sent to standard input line_call=l - Call lambda l with one argument containing a line which arrived on stdout each time that happens. If provided, the 'stdout' return value will be None. timeout=n - Wait n seconds for the program to finish, otherwise kill it. timeout_ok - True to prevent timeouts from being treated as errors. fail_message=s - Exit program and include string s if program fails. env=h - Pass environment hash 'h' to the child process, using the existing environment if the value is None. env_add=h - Add contents of hash 'h' to environment. Return Values: status - Status code returned by the program stdout - Contents of standard output as a single string stderr - Contents of standard erroras a single string """ process = None if [arg for arg in argv if arg is None]: raise Exception("Can't run with null arguments.") # Build up a new, incorruptable copy of the environment for the # child process to use. if env_add is None: env_add = {} if env is None and len(env_add) == 0: new_env = None else: new_env = (os.environ if env is None else env).copy() new_env.update(env_add) def __get_process(argv, new_env, attempts): """Try to start a process, handling EAGAINs.""" while attempts > 0: attempts -= 1 try: return _Popen(argv, stdin=subprocess32.PIPE, stdout=subprocess32.PIPE, stderr=subprocess32.PIPE, env=new_env) except OSError as ex: # Non-EAGAIN or last attempt gets re-raised. if ex.errno != errno.EAGAIN or attempts == 0: raise ex # TODO: Should we sleep a bit here? assert False, "This code should not be reached." try: process = __get_process(argv, new_env, attempts) __running_add(process) if line_call is None: # Single-shot I/O with optional timeout try: stdout, stderr = process.communicate(stdin, timeout=timeout) status = process.returncode except subprocess32.TimeoutExpired: _end_process(process) status = 0 if timeout_ok else 2 stdout = '' stderr = "Process took too long to run." else: # Read one line at a time, passing each to the line_call lambda if not isinstance(line_call, type(lambda: 0)): raise ValueError("Function provided is not a lambda.") if stdin is not None: process.stdin.write(stdin) process.stdin.close() stderr = '' stdout_fileno = process.stdout.fileno() stderr_fileno = process.stderr.fileno() fds = [stdout_fileno, stderr_fileno] if timeout is not None: end_time = pscheduler.time_now() \ + pscheduler.seconds_as_timedelta(timeout) else: time_left = None while True: if timeout is not None: time_left = pscheduler.timedelta_as_seconds( end_time - pscheduler.time_now()) reads, _, _ = polled_select(fds, [], [], time_left) if len(reads) == 0: __running_drop(process) _end_process(process) return 2, None, "Process took too long to run." for readfd in reads: if readfd == stdout_fileno: got_line = process.stdout.readline() if got_line != '': line_call(got_line[:-1]) elif readfd == stderr_fileno: got_line = process.stderr.readline() if got_line != '': stderr += got_line if process.poll() != None: break # Siphon off anything left on stdout while True: got_line = process.stdout.readline() if got_line == '': break line_call(got_line[:-1]) process.wait() status = process.returncode stdout = None except Exception as ex: extype, _, trace = sys.exc_info() status = 2 stdout = '' stderr = ''.join(traceback.format_exception_only(extype, ex)) \ + ''.join(traceback.format_exception(extype, ex, trace)).strip() if process is not None: __running_drop(process) _end_process(process) if fail_message is not None and status != 0: pscheduler.fail("%s: %s" % (fail_message, stderr)) return status, stdout, stderr
def run_program(argv, # Program name and args stdin=None, # What to send to stdin line_call=None, # Lambda to call when a line arrives timeout=None, # Seconds timeout_ok=False, # Treat timeouts as not being an error short=False, # True to force timeout to 2 seconds fail_message=None # Exit with this failure message ): """ Run a program and return the results. Arguments: argv - Array containing arguments, including name of program stdin=s - String containing what should be sent to standard input line_call=l - Call lambda l with one argument containing a line which arrived on stdout each time that happens. If provided, the 'stdout' return value will be None. timeout=n - Wait n seconds for the program to finish, otherwise kill it. timeout_ok - True to prevent timeouts from being treated as errors. short - True to force timeout to two seconds fail_message=s - Exit program and include string s if program fails. Return Values: status - Status code returned by the program stdout - Contents of standard output as a single string stderr - Contents of standard erroras a single string """ process = None if filter(lambda v: v is None, argv): raise Exception("Can't run with null arguments.") try: process = subprocess32.Popen(argv, stdin=subprocess32.PIPE, stdout=subprocess32.PIPE, stderr=subprocess32.PIPE, ) __running_add(process) if line_call is None: # Single-shot I/O with optional timeout try: stdout, stderr = process.communicate(stdin, timeout=timeout) status = process.returncode except subprocess32.TimeoutExpired: # Clean up after a timeout try: process.kill() except OSError: pass # Can't kill things that change UID process.communicate() status = 0 if timeout_ok else 2 # TODO: See if the exception has the contents of stdout and # stderr available. stdout = '' stderr = "Process took too long to run." else: # Read one line at a time, passing each to the line_call lambda if not isinstance(line_call, type(lambda:0)): raise ValueError("Function provided is not a lambda.") if stdin is not None: process.stdin.write(stdin) process.stdin.close() stderr = '' stdout_fileno = process.stdout.fileno() stderr_fileno = process.stderr.fileno() fds = [ stdout_fileno, stderr_fileno ] end_time = pscheduler.time_now() \ + pscheduler.seconds_as_timedelta(timeout) while True: time_left = pscheduler.timedelta_as_seconds( end_time - pscheduler.time_now() ) reads, writes, specials = select.select(fds, [], [], time_left) if len(reads) == 0: __running_drop(process) return 2, None, "Process took too long to run." for fd in reads: if fd == stdout_fileno: line = process.stdout.readline() if line != '': line_call(line[:-1]) elif fd == stderr_fileno: line = process.stderr.readline() if line != '': stderr += line if process.poll() != None: break process.wait() status = process.returncode stdout = None except Exception as ex: extype, ex_dummy, tb = sys.exc_info() status = 2 stdout = '' stderr = ''.join(traceback.format_exception_only(extype, ex)) \ + ''.join(traceback.format_exception(extype, ex, tb)).strip() if process is not None: __running_drop(process) if fail_message is not None and status != 0: pscheduler.fail("%s: %s" % (fail_message, stderr)) return status, stdout, stderr