Exemplo n.º 1
0
def json_load(source=None, exit_on_error=False, strip=True):
    """
    Load a blob of JSON and exit with failure if it didn't read.

    Arguments:

    source - String or open file containing the JSON.  If not
    specified, sys.stdin will be used.

    exit_on_error - Use the pScheduler failure mechanism to complain and
    exit the program.  (Default False)

    strip - Remove all pairs whose names begin with '#'.  This is a
    low-budget way to support comments wthout requiring a parser that
    understands them.  (Default True)
    """
    if source is None:
        source = sys.stdin

    try:
        if type(source) is str or type(source) is unicode:
            json_in = loads(str(source))
        elif type(source) is file:
            json_in = load(source)
        else:
            raise Exception("Internal error: bad source type ", type(source))
    except ValueError as ex:
        # TODO: Make this consistent and fix scripts that use it.
        if type(source) is str or not exit_on_error:
            raise ValueError("Invalid JSON: " + str(ex))
        else:
            pscheduler.fail("Invalid JSON: " + str(ex))

    return json_decomment(json_in) if strip else json_in
Exemplo n.º 2
0
def load_schema(schema_file):
    #find schema file
    schema_path = os.path.dirname(sys.argv[0])  
    if schema_path and not schema_path.endswith(os.sep):
        schema_path += os.sep
    try:
        schema = json.loads(open(schema_path + schema_file).read())
    except Exception, e:
        pscheduler.fail("Problem loading schema file: %s" % e)
Exemplo n.º 3
0
 def add_additional_metadata(self, test_spec={}):
     #this should not happen
     if not self.test_type:
         pscheduler.fail("Developer error. The test type must be set if storing a raw record.")
         
     for field in test_spec:
         key = "pscheduler-%s-%s" % (self.test_type, field)
         val = test_spec[field]
         self._parse_test_spec_field(key, val)
Exemplo n.º 4
0
def handle_storage_error(result, attempts=0, policy=[]):
    #build object
    retry = False
    archive_err_result = { 'succeeded': False, 'error': result }
    policy_attempt_sum = 0
    for p in policy:
        policy_attempt_sum += p['attempts']
        if policy_attempt_sum > attempts:
            retry = True
            archive_err_result['retry'] = p['wait']
            break
    if retry:
        pscheduler.succeed_json(archive_err_result)
    else:
        pscheduler.fail("Archiver permanently abandoned registering test after %d attempt(s): %s" % (attempts+1, result))
Exemplo n.º 5
0
def stat_control_pause():
    cursor = dbcursor_query("""
        SELECT
            control_is_paused(),
            date_trunc('second', pause_runs_until - now()),
            pause_runs_until = tstz_infinity()
        FROM control""")
    if cursor.rowcount != 1:
        pscheduler.fail("Got back more data than expected.")
    (is_paused, left, infinite) = cursor.fetchone()
    cursor.close()

    result = {"is_paused": is_paused}
    if is_paused:
        result["infinite"] = infinite
        if not infinite:
            result["remaining"] = pscheduler.timedelta_as_iso8601(left)

    return ok_json(result)
Exemplo n.º 6
0
def json_load(source=None, exit_on_error=False, strip=True, max_schema=None):
    """
    Load a blob of JSON and exit with failure if it didn't read.

    Arguments:

    source - String or open file containing the JSON.  If not
    specified, sys.stdin will be used.

    exit_on_error - Use the pScheduler failure mechanism to complain and
    exit the program.  (Default False)

    strip - Remove all pairs whose names begin with '#'.  This is a
    low-budget way to support comments wthout requiring a parser that
    understands them.  (Default True)

    max_schema - Check for a "schema" of no more than this integer value.
    """
    if source is None:
        source = sys.stdin

    try:
        if type(source) is str or type(source) is unicode:
            json_in = loads(str(source))
        elif type(source) is file:
            json_in = load(source)
        else:
            raise Exception("Internal error: bad source type ", type(source))
    except ValueError as ex:
        # TODO: Make this consistent and fix scripts that use it.
        if type(source) is str or not exit_on_error:
            raise ValueError("Invalid JSON: " + str(ex))
        else:
            pscheduler.fail("Invalid JSON: " + str(ex))

    if max_schema is not None:
        json_check_schema(json_in, max_schema)

    return json_decomment(json_in) if strip else json_in
Exemplo n.º 7
0
def run_program(
    argv,  # Program name and args
    stdin=None,  # What to send to stdin
    line_call=None,  # Lambda to call when a line arrives
    timeout=None,  # Seconds
    timeout_ok=False,  # Treat timeouts as not being an error
    fail_message=None,  # Exit with this failure message
    env=None,  # Environment for new process, None=existing
    env_add=None,  # Add hash to existing environment
    attempts=10):  # Max attempts to start the process
    """
    Run a program and return the results.

    Arguments:

    argv - Array containing arguments, including name of program
    stdin=s - String containing what should be sent to standard input
    line_call=l - Call lambda l with one argument containing a line which
        arrived on stdout each time that happens.  If provided, the
        'stdout' return value will be None.
    timeout=n - Wait n seconds for the program to finish, otherwise kill it.
    timeout_ok - True to prevent timeouts from being treated as errors.
    fail_message=s - Exit program and include string s if program fails.
    env=h - Pass environment hash 'h' to the child process, using the
        existing environment if the value is None.
    env_add=h - Add contents of hash 'h' to environment.


    Return Values:

    status - Status code returned by the program
    stdout - Contents of standard output as a single string
    stderr - Contents of standard erroras a single string
    """

    process = None

    if [arg for arg in argv if arg is None]:
        raise Exception("Can't run with null arguments.")

    # Build up a new, incorruptable copy of the environment for the
    # child process to use.

    if env_add is None:
        env_add = {}

    if env is None and len(env_add) == 0:
        new_env = None
    else:
        new_env = (os.environ if env is None else env).copy()
        new_env.update(env_add)

    def __get_process(argv, new_env, attempts):
        """Try to start a process, handling EAGAINs."""
        while attempts > 0:
            attempts -= 1
            try:
                return _Popen(argv,
                              stdin=subprocess32.PIPE,
                              stdout=subprocess32.PIPE,
                              stderr=subprocess32.PIPE,
                              env=new_env)
            except OSError as ex:
                # Non-EAGAIN or last attempt gets re-raised.
                if ex.errno != errno.EAGAIN or attempts == 0:
                    raise ex
                # TODO: Should we sleep a bit here?

        assert False, "This code should not be reached."

    try:
        process = __get_process(argv, new_env, attempts)

        __running_add(process)

        if line_call is None:

            # Single-shot I/O with optional timeout

            try:
                stdout, stderr = process.communicate(stdin, timeout=timeout)
                status = process.returncode

            except subprocess32.TimeoutExpired:
                _end_process(process)
                status = 0 if timeout_ok else 2
                stdout = ''
                stderr = "Process took too long to run."

        else:

            # Read one line at a time, passing each to the line_call lambda

            if not isinstance(line_call, type(lambda: 0)):
                raise ValueError("Function provided is not a lambda.")

            if stdin is not None:
                process.stdin.write(stdin)
            process.stdin.close()

            stderr = ''

            stdout_fileno = process.stdout.fileno()
            stderr_fileno = process.stderr.fileno()

            fds = [stdout_fileno, stderr_fileno]

            if timeout is not None:
                end_time = pscheduler.time_now() \
                    + pscheduler.seconds_as_timedelta(timeout)
            else:
                time_left = None

            while True:

                if timeout is not None:
                    time_left = pscheduler.timedelta_as_seconds(
                        end_time - pscheduler.time_now())

                reads, _, _ = polled_select(fds, [], [], time_left)

                if len(reads) == 0:
                    __running_drop(process)
                    _end_process(process)
                    return 2, None, "Process took too long to run."

                for readfd in reads:
                    if readfd == stdout_fileno:
                        got_line = process.stdout.readline()
                        if got_line != '':
                            line_call(got_line[:-1])
                    elif readfd == stderr_fileno:
                        got_line = process.stderr.readline()
                        if got_line != '':
                            stderr += got_line

                if process.poll() != None:
                    break

            # Siphon off anything left on stdout
            while True:
                got_line = process.stdout.readline()
                if got_line == '':
                    break
                line_call(got_line[:-1])

            process.wait()

            status = process.returncode
            stdout = None

    except Exception as ex:
        extype, _, trace = sys.exc_info()
        status = 2
        stdout = ''
        stderr = ''.join(traceback.format_exception_only(extype, ex)) \
            + ''.join(traceback.format_exception(extype, ex, trace)).strip()

    if process is not None:
        __running_drop(process)
        _end_process(process)

    if fail_message is not None and status != 0:
        pscheduler.fail("%s: %s" % (fail_message, stderr))

    return status, stdout, stderr
Exemplo n.º 8
0
def run_program(argv,              # Program name and args
                stdin=None,        # What to send to stdin
                line_call=None,    # Lambda to call when a line arrives
                timeout=None,      # Seconds
                timeout_ok=False,  # Treat timeouts as not being an error
                short=False,       # True to force timeout to 2 seconds
                fail_message=None  # Exit with this failure message
    ):
    """
    Run a program and return the results.

    Arguments:

    argv - Array containing arguments, including name of program
    stdin=s - String containing what should be sent to standard input
    line_call=l - Call lambda l with one argument containing a line which
        arrived on stdout each time that happens.  If provided, the
        'stdout' return value will be None.
    timeout=n - Wait n seconds for the program to finish, otherwise kill it.
    timeout_ok - True to prevent timeouts from being treated as errors.
    short - True to force timeout to two seconds
    fail_message=s - Exit program and include string s if program fails.

    Return Values:

    status - Status code returned by the program
    stdout - Contents of standard output as a single string
    stderr - Contents of standard erroras a single string
    """

    process = None

    if filter(lambda v: v is None, argv):
        raise Exception("Can't run with null arguments.")

    try:
        process = subprocess32.Popen(argv,
                                     stdin=subprocess32.PIPE,
                                     stdout=subprocess32.PIPE,
                                     stderr=subprocess32.PIPE,
                                     )

        __running_add(process)

        if line_call is None:

            # Single-shot I/O with optional timeout

            try:
                stdout, stderr = process.communicate(stdin, timeout=timeout)
                status = process.returncode

            except subprocess32.TimeoutExpired:
                # Clean up after a timeout
                try:
                    process.kill()
                except OSError:
                    pass  # Can't kill things that change UID
                process.communicate()

                status = 0 if timeout_ok else 2

                # TODO: See if the exception has the contents of stdout and
                # stderr available.
                stdout = ''
                stderr = "Process took too long to run."

        else:

            # Read one line at a time, passing each to the line_call lambda

            if not isinstance(line_call, type(lambda:0)):
                raise ValueError("Function provided is not a lambda.")

            if stdin is not None:
                process.stdin.write(stdin)
            process.stdin.close()

            stderr = ''

            stdout_fileno = process.stdout.fileno()
            stderr_fileno = process.stderr.fileno()

            fds = [ stdout_fileno, stderr_fileno ]

            end_time = pscheduler.time_now() \
                + pscheduler.seconds_as_timedelta(timeout)

            while True:

                time_left = pscheduler.timedelta_as_seconds(
                    end_time - pscheduler.time_now() )

                reads, writes, specials = select.select(fds, [], [], time_left)

                if len(reads) == 0:
                    __running_drop(process)
                    return 2, None, "Process took too long to run."

                for fd in reads:
                    if fd == stdout_fileno:
                        line = process.stdout.readline()
                        if line != '':
                            line_call(line[:-1])
                    elif fd == stderr_fileno:
                        line = process.stderr.readline()
                        if line != '':
                            stderr += line

                if process.poll() != None:
                    break

            process.wait()

            status = process.returncode
            stdout = None

    except Exception as ex:
        extype, ex_dummy, tb = sys.exc_info()
        status = 2
        stdout = ''
        stderr = ''.join(traceback.format_exception_only(extype, ex)) \
            + ''.join(traceback.format_exception(extype, ex, tb)).strip()


    if process is not None:
        __running_drop(process)

    if fail_message is not None and status != 0:
        pscheduler.fail("%s: %s" % (fail_message, stderr))

    return status, stdout, stderr