Ejemplo n.º 1
0
    def __init__(
            self,
            data  # Data suitable for this class
    ):

        valid, message = urlfetch_data_is_valid(data)
        if not valid:
            raise ValueError("Invalid data: %s" % message)

        self.url = data["url"]
        self.url_transform = _jq_filter(data.get("url-transform", None))
        self.bind = data.get("bind", None)
        self.follow = data.get("follow-redirects", True)
        self.timeout = pscheduler.timedelta_as_seconds(
            pscheduler.iso8601_as_timedelta(data.get("timeout", "PT3S")))
        self.verify = data.get("verify-keys", True)
        self.fail_result = data.get("fail-result", False)

        self.headers = data.get("headers", {})
        self.headers_transform = _jq_filter(data.get("headers-transform",
                                                     None))

        self.params = data.get("params", {})
        self.params_transform = _jq_filter(data.get("params-transform", None))

        self.success_only = data.get("success-only", False)
Ejemplo n.º 2
0
    def __init__(self,
                 data   # Data suitable for this class
                 ):

        valid, message = data_is_valid(data)
        if not valid:
            raise ValueError("Invalid data: %s" % message)

        self.exclude = [ ipaddr.IPNetwork(addr)
                         for addr in data['exclude'] ]

        try:
            timeout = pscheduler.iso8601_as_timedelta(data['timeout'])
            self.timeout = pscheduler.timedelta_as_seconds(timeout)
        except KeyError:
            self.timeout = 2

        try:
            self.fail_result = data['fail-result']
        except KeyError:
            self.fail_result = False

        self.resolver = dns.resolver.Resolver()
        self.resolver.timeout = self.timeout
        self.resolver.lifetime = self.timeout
Ejemplo n.º 3
0
    def __init__(
            self,
            data  # Data suitable for this class
    ):

        valid, message = data_is_valid(data)
        if not valid:
            raise ValueError("Invalid data: %s" % message)

        self.exclude = [ipaddr.IPNetwork(addr) for addr in data['exclude']]

        try:
            timeout = pscheduler.iso8601_as_timedelta(data['timeout'])
            self.timeout = pscheduler.timedelta_as_seconds(timeout)
        except KeyError:
            self.timeout = 2

        try:
            self.fail_result = data['fail-result']
        except KeyError:
            self.fail_result = False

        self.resolver = dns.resolver.Resolver()
        self.resolver.timeout = self.timeout
        self.resolver.lifetime = self.timeout
Ejemplo n.º 4
0
def safe_run(function,
             name=None,
             backoff=0.25,     # Backoff time increment
             backoff_max=60,   # Longest allowable backoff
             restart=True      # Call again if the function returns
             ):

    """
    Safely call a long-running lambda (usually a main program),
    catching and logging exceptions.  The lambda will be re-called
    immediately if it simply returns or after a linearly-increasing
    backoff if it raises an exception.  The backoff always applies if
    the function has not yet run successfully and time will reset once
    the lambda runs any longer than the last backoff delay.
    """

    if not isinstance(function, type(lambda:0)):
        raise ValueError("Function provided is not a lambda.")

    log = pscheduler.Log(name=name,
                         prefix='safe_run',
                         signals=False,
                         quiet=True
                         )

    initial_backoff = backoff
    current_backoff = backoff
    runs = 0

    while True:

        try:
            started = pscheduler.time_now()
            function()
            runs += 1
            if not restart:
                break

        except KeyboardInterrupt:
            break

        except Exception as ex:
            ran = pscheduler.time_now() - started
            ran_seconds = pscheduler.timedelta_as_seconds(ran)

            log.error("Program threw an exception after %s", ran)
            log.exception()

            # Running longer than the backoff is a good excuse to try
            # starting over.
            if ran_seconds > current_backoff and runs != 0:
                currrent_backoff = initial_backoff
                log.error("Restarting immediately.")
            else:
                log.error("Waiting %s seconds before restarting", current_backoff)
                time.sleep(current_backoff)
                if current_backoff < backoff_max:
                    current_backoff += initial_backoff
                log.error("Restarting")
Ejemplo n.º 5
0
def safe_run(
    function,
    name=None,
    backoff=0.25,  # Backoff time increment
    backoff_max=60,  # Longest allowable backoff
    restart=True  # Call again if the function returns
):
    """
    Safely call a long-running lambda (usually a main program),
    catching and logging exceptions.  The lambda will be re-called
    immediately if it simply returns or after a linearly-increasing
    backoff if it raises an exception.  The backoff always applies if
    the function has not yet run successfully and time will reset once
    the lambda runs any longer than the last backoff delay.
    """

    if not isinstance(function, type(lambda: 0)):
        raise ValueError("Function provided is not a lambda.")

    log = pscheduler.Log(name=name,
                         prefix='safe_run',
                         signals=False,
                         quiet=True)

    initial_backoff = backoff
    current_backoff = backoff
    runs = 0

    while True:

        try:
            started = pscheduler.time_now()
            function()
            runs += 1
            if not restart:
                break

        except KeyboardInterrupt:
            break

        except Exception as ex:
            ran = pscheduler.time_now() - started
            ran_seconds = pscheduler.timedelta_as_seconds(ran)

            log.error("Program threw an exception after %s", ran)
            log.exception()

            # Running longer than the backoff is a good excuse to try
            # starting over.
            if ran_seconds > current_backoff and runs != 0:
                currrent_backoff = initial_backoff
                log.error("Restarting immediately.")
            else:
                log.error("Waiting %s seconds before restarting",
                          current_backoff)
                time.sleep(current_backoff)
                if current_backoff < backoff_max:
                    current_backoff += initial_backoff
                log.error("Restarting")
Ejemplo n.º 6
0
    def __init__(self,
                 data   # Data suitable for this class
                 ):

        valid, message = data_is_valid(data)
        if not valid:
            raise ValueError("Invalid data: %s" % message)

        self.matcher = pscheduler.StringMatcher(data['match'])

        timeout = pscheduler.timedelta_as_seconds(
            pscheduler.iso8601_as_timedelta(data['timeout']))

        self.resolver = dns.resolver.Resolver()
        self.resolver.timeout = timeout
        self.resolver.lifetime = timeout
Ejemplo n.º 7
0
    def __init__(self,
                 data   # Data suitable for this class
                 ):

        valid, message = data_is_valid(data)
        if not valid:
            raise ValueError("Invalid data: %s" % message)

        self.matcher = pscheduler.StringMatcher(data['match'])

        timeout = pscheduler.timedelta_as_seconds(
            pscheduler.iso8601_as_timedelta(data['timeout']))

        self.resolver = dns.resolver.Resolver()
        self.resolver.timeout = timeout
        self.resolver.lifetime = timeout
Ejemplo n.º 8
0
def setup_time(rtt):

    """Setup time: How long it takes for iperf2 to set up a test.  This is
    heavily influenced by the RTT between the hosts involved.
    Measurements taken as part of #868 indicate that every 50 ms of
    latency adds about one second.  Previously, this was done with a
    static fudge factor of four seconds, which has been set as the
    default.  RTT has no influence on how long it takes to run the
    test; iperf2 will stop after the specified number of seconds
    regardless of how much data was transferred.
    The rtt is an ISO8601 duration as a string.  If None, the default
    will be used.
    Returned value is seconds, minimum 1.
    """

    delta = pscheduler.iso8601_as_timedelta(rtt or DEFAULT_LINK_RTT)
    rtt = pscheduler.timedelta_as_seconds(delta)

    # TODO: This is the same as what we do for iperf3 but may need tuning.
    return max(((rtt * 1000.0) / 50.0), 1.0)
Ejemplo n.º 9
0
def safe_run(
    function,
    name=None,
    backoff=0.25,  # Backoff time increment
    backoff_max=60.0,  # Longest allowable backoff
    restart=True  # Call again if the function returns
):
    """
    Safely call a long-running lambda (usually a main program),
    catching and logging exceptions.  If an exception is thrown, the
    calling program will be re-exec'd using the same arguments
    immediately if it simply returns or after a linearly-increasing
    backoff if it raises an exception.  The backoff always applies if
    the function has not yet run successfully and time will reset once
    the lambda runs any longer than the last backoff delay.
    """

    if not isinstance(function, type(lambda: 0)):
        raise ValueError("Function provided is not a lambda.")

    log = pscheduler.Log(name=name,
                         prefix='safe_run',
                         signals=False,
                         quiet=True)

    # Inherit state from the environment

    if STATE_VARIABLE in os.environ:

        try:
            depickled = pickle.loads(os.environ[STATE_VARIABLE])

            initial_backoff = depickled['initial_backoff']
            assert type(initial_backoff) in [int, float]

            current_backoff = depickled['current_backoff']
            assert type(current_backoff) in [int, float]

            runs = depickled['runs']
            assert type(runs) == int

        except Exception as ex:
            log.error("Failed to decode %s '%s': %s" %
                      (STATE_VARIABLE, os.environ[STATE_VARIABLE], ex))
            exit(1)
    else:

        initial_backoff = backoff
        current_backoff = backoff
        runs = 0

    # Run the function

    do_restart = False

    try:
        started = pscheduler.time_now()
        function()
        runs += 1
        do_restart = restart

    except KeyboardInterrupt:
        pass

    except Exception as ex:
        ran = pscheduler.time_now() - started
        ran_seconds = pscheduler.timedelta_as_seconds(ran)

        log.error("Program threw an exception after %s", ran)
        log.exception()

        # Running longer than the backoff is a good excuse to try
        # starting over.
        if ran_seconds > current_backoff and runs != 0:
            currrent_backoff = initial_backoff
        else:
            log.error("Waiting %s seconds before restarting", current_backoff)
            time.sleep(current_backoff)
            if current_backoff < backoff_max:
                current_backoff += initial_backoff

        do_restart = True

    if not do_restart:
        log.error("Exiting")
        exit(0)

    log.error("Restarting: %s", sys.argv)

    #
    # Pickle the current state to pass along
    #

    to_pickle = {
        'initial_backoff': initial_backoff,
        'current_backoff': current_backoff,
        'runs': runs
    }
    os.environ[STATE_VARIABLE] = pickle.dumps(to_pickle)

    os.execvp(sys.argv[0], sys.argv)
Ejemplo n.º 10
0
def run_program(
    argv,  # Program name and args
    stdin=None,  # What to send to stdin
    line_call=None,  # Lambda to call when a line arrives
    timeout=None,  # Seconds
    timeout_ok=False,  # Treat timeouts as not being an error
    fail_message=None,  # Exit with this failure message
    env=None,  # Environment for new process, None=existing
    env_add=None,  # Add hash to existing environment
    attempts=10):  # Max attempts to start the process
    """
    Run a program and return the results.

    Arguments:

    argv - Array containing arguments, including name of program
    stdin=s - String containing what should be sent to standard input
    line_call=l - Call lambda l with one argument containing a line which
        arrived on stdout each time that happens.  If provided, the
        'stdout' return value will be None.
    timeout=n - Wait n seconds for the program to finish, otherwise kill it.
    timeout_ok - True to prevent timeouts from being treated as errors.
    fail_message=s - Exit program and include string s if program fails.
    env=h - Pass environment hash 'h' to the child process, using the
        existing environment if the value is None.
    env_add=h - Add contents of hash 'h' to environment.


    Return Values:

    status - Status code returned by the program
    stdout - Contents of standard output as a single string
    stderr - Contents of standard erroras a single string
    """

    process = None

    if [arg for arg in argv if arg is None]:
        raise Exception("Can't run with null arguments.")

    # Build up a new, incorruptable copy of the environment for the
    # child process to use.

    if env_add is None:
        env_add = {}

    if env is None and len(env_add) == 0:
        new_env = None
    else:
        new_env = (os.environ if env is None else env).copy()
        new_env.update(env_add)

    def __get_process(argv, new_env, attempts):
        """Try to start a process, handling EAGAINs."""
        while attempts > 0:
            attempts -= 1
            try:
                return _Popen(argv,
                              stdin=subprocess32.PIPE,
                              stdout=subprocess32.PIPE,
                              stderr=subprocess32.PIPE,
                              env=new_env)
            except OSError as ex:
                # Non-EAGAIN or last attempt gets re-raised.
                if ex.errno != errno.EAGAIN or attempts == 0:
                    raise ex
                # TODO: Should we sleep a bit here?

        assert False, "This code should not be reached."

    try:
        process = __get_process(argv, new_env, attempts)

        __running_add(process)

        if line_call is None:

            # Single-shot I/O with optional timeout

            try:
                stdout, stderr = process.communicate(stdin, timeout=timeout)
                status = process.returncode

            except subprocess32.TimeoutExpired:
                _end_process(process)
                status = 0 if timeout_ok else 2
                stdout = ''
                stderr = "Process took too long to run."

        else:

            # Read one line at a time, passing each to the line_call lambda

            if not isinstance(line_call, type(lambda: 0)):
                raise ValueError("Function provided is not a lambda.")

            if stdin is not None:
                process.stdin.write(stdin)
            process.stdin.close()

            stderr = ''

            stdout_fileno = process.stdout.fileno()
            stderr_fileno = process.stderr.fileno()

            fds = [stdout_fileno, stderr_fileno]

            if timeout is not None:
                end_time = pscheduler.time_now() \
                    + pscheduler.seconds_as_timedelta(timeout)
            else:
                time_left = None

            while True:

                if timeout is not None:
                    time_left = pscheduler.timedelta_as_seconds(
                        end_time - pscheduler.time_now())

                reads, _, _ = polled_select(fds, [], [], time_left)

                if len(reads) == 0:
                    __running_drop(process)
                    _end_process(process)
                    return 2, None, "Process took too long to run."

                for readfd in reads:
                    if readfd == stdout_fileno:
                        got_line = process.stdout.readline()
                        if got_line != '':
                            line_call(got_line[:-1])
                    elif readfd == stderr_fileno:
                        got_line = process.stderr.readline()
                        if got_line != '':
                            stderr += got_line

                if process.poll() != None:
                    break

            # Siphon off anything left on stdout
            while True:
                got_line = process.stdout.readline()
                if got_line == '':
                    break
                line_call(got_line[:-1])

            process.wait()

            status = process.returncode
            stdout = None

    except Exception as ex:
        extype, _, trace = sys.exc_info()
        status = 2
        stdout = ''
        stderr = ''.join(traceback.format_exception_only(extype, ex)) \
            + ''.join(traceback.format_exception(extype, ex, trace)).strip()

    if process is not None:
        __running_drop(process)
        _end_process(process)

    if fail_message is not None and status != 0:
        pscheduler.fail("%s: %s" % (fail_message, stderr))

    return status, stdout, stderr
Ejemplo n.º 11
0
def run(input):

    # TODO: Check the spec schema

    try:
        assert input["test"]["type"] == "http"
        source = input['test']['spec']['url']
        always_succeed = input['test']['spec'].get('always-succeed', False)
        keep_content = input['test']['spec'].get('keep-content', None)
        timeout_iso = input['test']['spec'].get('timeout', 'PT10S')
        ip_version = input['test']['spec'].get('ip-version', None)
        timeout = pscheduler.timedelta_as_seconds(
            pscheduler.iso8601_as_timedelta(timeout_iso))
    except KeyError as ex:
        return ({"succeeded": False, "error": "Missing data in input"})

    # Can-run should weed these out, but backstop it with a check just in case.
    if source[0:5] == "file:" and keep_content is not None:
        return ({
            "succeeded": False,
            "error": "Cannot keep content from file:// URLs",
            "diags": None
        })

    succeeded = False
    error = None
    diags = []

    STDERR = ""

    # TODO: Implement this with libcurl

    curl = pycurl.Curl()

    curl.setopt(pycurl.URL, str(source))

    # TODO: This test doesn't have bind but needs one.
    # curl.setopt(pycurl.INTERFACE, str(bind))

    # TODO: Redirects as an option?
    # curl.setopt(pycurl.FOLLOWLOCATION, allow_redirects)

    if timeout is not None:
        curl.setopt(pycurl.TIMEOUT_MS, int(timeout * 1000.0))

    curl.setopt(pycurl.SSL_VERIFYHOST, False)
    curl.setopt(pycurl.SSL_VERIFYPEER, False)

    if ip_version is not None:
        curl.setopt(
            pycurl.IPRESOLVE,
            pycurl.IPRESOLVE_V4 if ip_version == 4 else pycurl.IPRESOLVE_V6)

    buf = io.BytesIO()
    curl.setopt(pycurl.WRITEFUNCTION, buf.write)

    text = ""

    try:
        start_time = datetime.datetime.now()
        curl.perform()
        status = curl.getinfo(pycurl.HTTP_CODE)
        # PycURL returns a zero for non-HTTP URLs
        if status == 0:
            status = 200
        text = buf.getvalue().decode()
    except pycurl.error as ex:
        code, message = ex.args
        status = 400
        text = message
    finally:
        end_time = datetime.datetime.now()
        curl.close()
        buf.close()

    # 200-299 is success; anything else is an error.
    fetch_succeeded = (status >= 200 and status < 300)
    succeeded = always_succeed or fetch_succeeded

    if succeeded:

        schema = pscheduler.HighInteger(1)

        run_result = {
            "succeeded": True,
            "time": pscheduler.timedelta_as_iso8601(end_time - start_time)
        }

        if always_succeed:
            run_result["status"] = status
            schema.set(2)

        try:
            run_result["found"] = text.find(
                input['test']['spec']["parse"]) >= 0
        except KeyError:
            pass

        # If the fetch failed or we've been told to keep 0 content, plaster it all in.
        if (not fetch_succeeded) or (keep_content is not None
                                     and keep_content == 0):
            run_result["content"] = text
            schema.set(2)
        elif keep_content is not None:
            run_result["content"] = text[:keep_content]

            schema.set(2)

        run_result["schema"] = schema.value()

        return {
            "succeeded": True,
            "diags": None,
            "error": None,
            "result": run_result
        }

    else:

        return {
            "succeeded": False,
            "diags": "Fetch returned non-success status %d" % (status),
            "error": text
        }

    assert False, "Should not be reached."
Ejemplo n.º 12
0
def run_program(argv,              # Program name and args
                stdin=None,        # What to send to stdin
                line_call=None,    # Lambda to call when a line arrives
                timeout=None,      # Seconds
                timeout_ok=False,  # Treat timeouts as not being an error
                short=False,       # True to force timeout to 2 seconds
                fail_message=None  # Exit with this failure message
    ):
    """
    Run a program and return the results.

    Arguments:

    argv - Array containing arguments, including name of program
    stdin=s - String containing what should be sent to standard input
    line_call=l - Call lambda l with one argument containing a line which
        arrived on stdout each time that happens.  If provided, the
        'stdout' return value will be None.
    timeout=n - Wait n seconds for the program to finish, otherwise kill it.
    timeout_ok - True to prevent timeouts from being treated as errors.
    short - True to force timeout to two seconds
    fail_message=s - Exit program and include string s if program fails.

    Return Values:

    status - Status code returned by the program
    stdout - Contents of standard output as a single string
    stderr - Contents of standard erroras a single string
    """

    process = None

    if filter(lambda v: v is None, argv):
        raise Exception("Can't run with null arguments.")

    try:
        process = subprocess32.Popen(argv,
                                     stdin=subprocess32.PIPE,
                                     stdout=subprocess32.PIPE,
                                     stderr=subprocess32.PIPE,
                                     )

        __running_add(process)

        if line_call is None:

            # Single-shot I/O with optional timeout

            try:
                stdout, stderr = process.communicate(stdin, timeout=timeout)
                status = process.returncode

            except subprocess32.TimeoutExpired:
                # Clean up after a timeout
                try:
                    process.kill()
                except OSError:
                    pass  # Can't kill things that change UID
                process.communicate()

                status = 0 if timeout_ok else 2

                # TODO: See if the exception has the contents of stdout and
                # stderr available.
                stdout = ''
                stderr = "Process took too long to run."

        else:

            # Read one line at a time, passing each to the line_call lambda

            if not isinstance(line_call, type(lambda:0)):
                raise ValueError("Function provided is not a lambda.")

            if stdin is not None:
                process.stdin.write(stdin)
            process.stdin.close()

            stderr = ''

            stdout_fileno = process.stdout.fileno()
            stderr_fileno = process.stderr.fileno()

            fds = [ stdout_fileno, stderr_fileno ]

            end_time = pscheduler.time_now() \
                + pscheduler.seconds_as_timedelta(timeout)

            while True:

                time_left = pscheduler.timedelta_as_seconds(
                    end_time - pscheduler.time_now() )

                reads, writes, specials = select.select(fds, [], [], time_left)

                if len(reads) == 0:
                    __running_drop(process)
                    return 2, None, "Process took too long to run."

                for fd in reads:
                    if fd == stdout_fileno:
                        line = process.stdout.readline()
                        if line != '':
                            line_call(line[:-1])
                    elif fd == stderr_fileno:
                        line = process.stderr.readline()
                        if line != '':
                            stderr += line

                if process.poll() != None:
                    break

            process.wait()

            status = process.returncode
            stdout = None

    except Exception as ex:
        extype, ex_dummy, tb = sys.exc_info()
        status = 2
        stdout = ''
        stderr = ''.join(traceback.format_exception_only(extype, ex)) \
            + ''.join(traceback.format_exception(extype, ex, tb)).strip()


    if process is not None:
        __running_drop(process)

    if fail_message is not None and status != 0:
        pscheduler.fail("%s: %s" % (fail_message, stderr))

    return status, stdout, stderr