def __init__(
self,
data # Data suitable for this class
):
valid, message = urlfetch_data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.url = data["url"]
self.url_transform = _jq_filter(data.get("url-transform", None))
self.bind = data.get("bind", None)
self.follow = data.get("follow-redirects", True)
self.timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(data.get("timeout", "PT3S")))
self.verify = data.get("verify-keys", True)
self.fail_result = data.get("fail-result", False)
self.headers = data.get("headers", {})
self.headers_transform = _jq_filter(data.get("headers-transform",
None))
self.params = data.get("params", {})
self.params_transform = _jq_filter(data.get("params-transform", None))
self.success_only = data.get("success-only", False)
def __init__(self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.exclude = [ ipaddr.IPNetwork(addr)
for addr in data['exclude'] ]
try:
timeout = pscheduler.iso8601_as_timedelta(data['timeout'])
self.timeout = pscheduler.timedelta_as_seconds(timeout)
except KeyError:
self.timeout = 2
try:
self.fail_result = data['fail-result']
except KeyError:
self.fail_result = False
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
def __init__(
self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.exclude = [ipaddr.IPNetwork(addr) for addr in data['exclude']]
try:
timeout = pscheduler.iso8601_as_timedelta(data['timeout'])
self.timeout = pscheduler.timedelta_as_seconds(timeout)
except KeyError:
self.timeout = 2
try:
self.fail_result = data['fail-result']
except KeyError:
self.fail_result = False
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
def safe_run(function,
name=None,
backoff=0.25, # Backoff time increment
backoff_max=60, # Longest allowable backoff
restart=True # Call again if the function returns
):
"""
Safely call a long-running lambda (usually a main program),
catching and logging exceptions. The lambda will be re-called
immediately if it simply returns or after a linearly-increasing
backoff if it raises an exception. The backoff always applies if
the function has not yet run successfully and time will reset once
the lambda runs any longer than the last backoff delay.
"""
if not isinstance(function, type(lambda:0)):
raise ValueError("Function provided is not a lambda.")
log = pscheduler.Log(name=name,
prefix='safe_run',
signals=False,
quiet=True
)
initial_backoff = backoff
current_backoff = backoff
runs = 0
while True:
try:
started = pscheduler.time_now()
function()
runs += 1
if not restart:
break
except KeyboardInterrupt:
break
except Exception as ex:
ran = pscheduler.time_now() - started
ran_seconds = pscheduler.timedelta_as_seconds(ran)
log.error("Program threw an exception after %s", ran)
log.exception()
# Running longer than the backoff is a good excuse to try
# starting over.
if ran_seconds > current_backoff and runs != 0:
currrent_backoff = initial_backoff
log.error("Restarting immediately.")
else:
log.error("Waiting %s seconds before restarting", current_backoff)
time.sleep(current_backoff)
if current_backoff < backoff_max:
current_backoff += initial_backoff
log.error("Restarting")
def safe_run(
function,
name=None,
backoff=0.25, # Backoff time increment
backoff_max=60, # Longest allowable backoff
restart=True # Call again if the function returns
):
"""
Safely call a long-running lambda (usually a main program),
catching and logging exceptions. The lambda will be re-called
immediately if it simply returns or after a linearly-increasing
backoff if it raises an exception. The backoff always applies if
the function has not yet run successfully and time will reset once
the lambda runs any longer than the last backoff delay.
"""
if not isinstance(function, type(lambda: 0)):
raise ValueError("Function provided is not a lambda.")
log = pscheduler.Log(name=name,
prefix='safe_run',
signals=False,
quiet=True)
initial_backoff = backoff
current_backoff = backoff
runs = 0
while True:
try:
started = pscheduler.time_now()
function()
runs += 1
if not restart:
break
except KeyboardInterrupt:
break
except Exception as ex:
ran = pscheduler.time_now() - started
ran_seconds = pscheduler.timedelta_as_seconds(ran)
log.error("Program threw an exception after %s", ran)
log.exception()
# Running longer than the backoff is a good excuse to try
# starting over.
if ran_seconds > current_backoff and runs != 0:
currrent_backoff = initial_backoff
log.error("Restarting immediately.")
else:
log.error("Waiting %s seconds before restarting",
current_backoff)
time.sleep(current_backoff)
if current_backoff < backoff_max:
current_backoff += initial_backoff
log.error("Restarting")
def __init__(self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.matcher = pscheduler.StringMatcher(data['match'])
timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(data['timeout']))
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = timeout
self.resolver.lifetime = timeout
def __init__(self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.matcher = pscheduler.StringMatcher(data['match'])
timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(data['timeout']))
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = timeout
self.resolver.lifetime = timeout
def setup_time(rtt):
"""Setup time: How long it takes for iperf2 to set up a test. This is
heavily influenced by the RTT between the hosts involved.
Measurements taken as part of #868 indicate that every 50 ms of
latency adds about one second. Previously, this was done with a
static fudge factor of four seconds, which has been set as the
default. RTT has no influence on how long it takes to run the
test; iperf2 will stop after the specified number of seconds
regardless of how much data was transferred.
The rtt is an ISO8601 duration as a string. If None, the default
will be used.
Returned value is seconds, minimum 1.
"""
delta = pscheduler.iso8601_as_timedelta(rtt or DEFAULT_LINK_RTT)
rtt = pscheduler.timedelta_as_seconds(delta)
# TODO: This is the same as what we do for iperf3 but may need tuning.
return max(((rtt * 1000.0) / 50.0), 1.0)
def safe_run(
function,
name=None,
backoff=0.25, # Backoff time increment
backoff_max=60.0, # Longest allowable backoff
restart=True # Call again if the function returns
):
"""
Safely call a long-running lambda (usually a main program),
catching and logging exceptions. If an exception is thrown, the
calling program will be re-exec'd using the same arguments
immediately if it simply returns or after a linearly-increasing
backoff if it raises an exception. The backoff always applies if
the function has not yet run successfully and time will reset once
the lambda runs any longer than the last backoff delay.
"""
if not isinstance(function, type(lambda: 0)):
raise ValueError("Function provided is not a lambda.")
log = pscheduler.Log(name=name,
prefix='safe_run',
signals=False,
quiet=True)
# Inherit state from the environment
if STATE_VARIABLE in os.environ:
try:
depickled = pickle.loads(os.environ[STATE_VARIABLE])
initial_backoff = depickled['initial_backoff']
assert type(initial_backoff) in [int, float]
current_backoff = depickled['current_backoff']
assert type(current_backoff) in [int, float]
runs = depickled['runs']
assert type(runs) == int
except Exception as ex:
log.error("Failed to decode %s '%s': %s" %
(STATE_VARIABLE, os.environ[STATE_VARIABLE], ex))
exit(1)
else:
initial_backoff = backoff
current_backoff = backoff
runs = 0
# Run the function
do_restart = False
try:
started = pscheduler.time_now()
function()
runs += 1
do_restart = restart
except KeyboardInterrupt:
pass
except Exception as ex:
ran = pscheduler.time_now() - started
ran_seconds = pscheduler.timedelta_as_seconds(ran)
log.error("Program threw an exception after %s", ran)
log.exception()
# Running longer than the backoff is a good excuse to try
# starting over.
if ran_seconds > current_backoff and runs != 0:
currrent_backoff = initial_backoff
else:
log.error("Waiting %s seconds before restarting", current_backoff)
time.sleep(current_backoff)
if current_backoff < backoff_max:
current_backoff += initial_backoff
do_restart = True
if not do_restart:
log.error("Exiting")
exit(0)
log.error("Restarting: %s", sys.argv)
#
# Pickle the current state to pass along
#
to_pickle = {
'initial_backoff': initial_backoff,
'current_backoff': current_backoff,
'runs': runs
}
os.environ[STATE_VARIABLE] = pickle.dumps(to_pickle)
os.execvp(sys.argv[0], sys.argv)
def run_program(
argv, # Program name and args
stdin=None, # What to send to stdin
line_call=None, # Lambda to call when a line arrives
timeout=None, # Seconds
timeout_ok=False, # Treat timeouts as not being an error
fail_message=None, # Exit with this failure message
env=None, # Environment for new process, None=existing
env_add=None, # Add hash to existing environment
attempts=10): # Max attempts to start the process
"""
Run a program and return the results.
Arguments:
argv - Array containing arguments, including name of program
stdin=s - String containing what should be sent to standard input
line_call=l - Call lambda l with one argument containing a line which
arrived on stdout each time that happens. If provided, the
'stdout' return value will be None.
timeout=n - Wait n seconds for the program to finish, otherwise kill it.
timeout_ok - True to prevent timeouts from being treated as errors.
fail_message=s - Exit program and include string s if program fails.
env=h - Pass environment hash 'h' to the child process, using the
existing environment if the value is None.
env_add=h - Add contents of hash 'h' to environment.
Return Values:
status - Status code returned by the program
stdout - Contents of standard output as a single string
stderr - Contents of standard erroras a single string
"""
process = None
if [arg for arg in argv if arg is None]:
raise Exception("Can't run with null arguments.")
# Build up a new, incorruptable copy of the environment for the
# child process to use.
if env_add is None:
env_add = {}
if env is None and len(env_add) == 0:
new_env = None
else:
new_env = (os.environ if env is None else env).copy()
new_env.update(env_add)
def __get_process(argv, new_env, attempts):
"""Try to start a process, handling EAGAINs."""
while attempts > 0:
attempts -= 1
try:
return _Popen(argv,
stdin=subprocess32.PIPE,
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
env=new_env)
except OSError as ex:
# Non-EAGAIN or last attempt gets re-raised.
if ex.errno != errno.EAGAIN or attempts == 0:
raise ex
# TODO: Should we sleep a bit here?
assert False, "This code should not be reached."
try:
process = __get_process(argv, new_env, attempts)
__running_add(process)
if line_call is None:
# Single-shot I/O with optional timeout
try:
stdout, stderr = process.communicate(stdin, timeout=timeout)
status = process.returncode
except subprocess32.TimeoutExpired:
_end_process(process)
status = 0 if timeout_ok else 2
stdout = ''
stderr = "Process took too long to run."
else:
# Read one line at a time, passing each to the line_call lambda
if not isinstance(line_call, type(lambda: 0)):
raise ValueError("Function provided is not a lambda.")
if stdin is not None:
process.stdin.write(stdin)
process.stdin.close()
stderr = ''
stdout_fileno = process.stdout.fileno()
stderr_fileno = process.stderr.fileno()
fds = [stdout_fileno, stderr_fileno]
if timeout is not None:
end_time = pscheduler.time_now() \
+ pscheduler.seconds_as_timedelta(timeout)
else:
time_left = None
while True:
if timeout is not None:
time_left = pscheduler.timedelta_as_seconds(
end_time - pscheduler.time_now())
reads, _, _ = polled_select(fds, [], [], time_left)
if len(reads) == 0:
__running_drop(process)
_end_process(process)
return 2, None, "Process took too long to run."
for readfd in reads:
if readfd == stdout_fileno:
got_line = process.stdout.readline()
if got_line != '':
line_call(got_line[:-1])
elif readfd == stderr_fileno:
got_line = process.stderr.readline()
if got_line != '':
stderr += got_line
if process.poll() != None:
break
# Siphon off anything left on stdout
while True:
got_line = process.stdout.readline()
if got_line == '':
break
line_call(got_line[:-1])
process.wait()
status = process.returncode
stdout = None
except Exception as ex:
extype, _, trace = sys.exc_info()
status = 2
stdout = ''
stderr = ''.join(traceback.format_exception_only(extype, ex)) \
+ ''.join(traceback.format_exception(extype, ex, trace)).strip()
if process is not None:
__running_drop(process)
_end_process(process)
if fail_message is not None and status != 0:
pscheduler.fail("%s: %s" % (fail_message, stderr))
return status, stdout, stderr
def run(input):
# TODO: Check the spec schema
try:
assert input["test"]["type"] == "http"
source = input['test']['spec']['url']
always_succeed = input['test']['spec'].get('always-succeed', False)
keep_content = input['test']['spec'].get('keep-content', None)
timeout_iso = input['test']['spec'].get('timeout', 'PT10S')
ip_version = input['test']['spec'].get('ip-version', None)
timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(timeout_iso))
except KeyError as ex:
return ({"succeeded": False, "error": "Missing data in input"})
# Can-run should weed these out, but backstop it with a check just in case.
if source[0:5] == "file:" and keep_content is not None:
return ({
"succeeded": False,
"error": "Cannot keep content from file:// URLs",
"diags": None
})
succeeded = False
error = None
diags = []
STDERR = ""
# TODO: Implement this with libcurl
curl = pycurl.Curl()
curl.setopt(pycurl.URL, str(source))
# TODO: This test doesn't have bind but needs one.
# curl.setopt(pycurl.INTERFACE, str(bind))
# TODO: Redirects as an option?
# curl.setopt(pycurl.FOLLOWLOCATION, allow_redirects)
if timeout is not None:
curl.setopt(pycurl.TIMEOUT_MS, int(timeout * 1000.0))
curl.setopt(pycurl.SSL_VERIFYHOST, False)
curl.setopt(pycurl.SSL_VERIFYPEER, False)
if ip_version is not None:
curl.setopt(
pycurl.IPRESOLVE,
pycurl.IPRESOLVE_V4 if ip_version == 4 else pycurl.IPRESOLVE_V6)
buf = io.BytesIO()
curl.setopt(pycurl.WRITEFUNCTION, buf.write)
text = ""
try:
start_time = datetime.datetime.now()
curl.perform()
status = curl.getinfo(pycurl.HTTP_CODE)
# PycURL returns a zero for non-HTTP URLs
if status == 0:
status = 200
text = buf.getvalue().decode()
except pycurl.error as ex:
code, message = ex.args
status = 400
text = message
finally:
end_time = datetime.datetime.now()
curl.close()
buf.close()
# 200-299 is success; anything else is an error.
fetch_succeeded = (status >= 200 and status < 300)
succeeded = always_succeed or fetch_succeeded
if succeeded:
schema = pscheduler.HighInteger(1)
run_result = {
"succeeded": True,
"time": pscheduler.timedelta_as_iso8601(end_time - start_time)
}
if always_succeed:
run_result["status"] = status
schema.set(2)
try:
run_result["found"] = text.find(
input['test']['spec']["parse"]) >= 0
except KeyError:
pass
# If the fetch failed or we've been told to keep 0 content, plaster it all in.
if (not fetch_succeeded) or (keep_content is not None
and keep_content == 0):
run_result["content"] = text
schema.set(2)
elif keep_content is not None:
run_result["content"] = text[:keep_content]
schema.set(2)
run_result["schema"] = schema.value()
return {
"succeeded": True,
"diags": None,
"error": None,
"result": run_result
}
else:
return {
"succeeded": False,
"diags": "Fetch returned non-success status %d" % (status),
"error": text
}
assert False, "Should not be reached."
def run_program(argv, # Program name and args
stdin=None, # What to send to stdin
line_call=None, # Lambda to call when a line arrives
timeout=None, # Seconds
timeout_ok=False, # Treat timeouts as not being an error
short=False, # True to force timeout to 2 seconds
fail_message=None # Exit with this failure message
):
"""
Run a program and return the results.
Arguments:
argv - Array containing arguments, including name of program
stdin=s - String containing what should be sent to standard input
line_call=l - Call lambda l with one argument containing a line which
arrived on stdout each time that happens. If provided, the
'stdout' return value will be None.
timeout=n - Wait n seconds for the program to finish, otherwise kill it.
timeout_ok - True to prevent timeouts from being treated as errors.
short - True to force timeout to two seconds
fail_message=s - Exit program and include string s if program fails.
Return Values:
status - Status code returned by the program
stdout - Contents of standard output as a single string
stderr - Contents of standard erroras a single string
"""
process = None
if filter(lambda v: v is None, argv):
raise Exception("Can't run with null arguments.")
try:
process = subprocess32.Popen(argv,
stdin=subprocess32.PIPE,
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
)
__running_add(process)
if line_call is None:
# Single-shot I/O with optional timeout
try:
stdout, stderr = process.communicate(stdin, timeout=timeout)
status = process.returncode
except subprocess32.TimeoutExpired:
# Clean up after a timeout
try:
process.kill()
except OSError:
pass # Can't kill things that change UID
process.communicate()
status = 0 if timeout_ok else 2
# TODO: See if the exception has the contents of stdout and
# stderr available.
stdout = ''
stderr = "Process took too long to run."
else:
# Read one line at a time, passing each to the line_call lambda
if not isinstance(line_call, type(lambda:0)):
raise ValueError("Function provided is not a lambda.")
if stdin is not None:
process.stdin.write(stdin)
process.stdin.close()
stderr = ''
stdout_fileno = process.stdout.fileno()
stderr_fileno = process.stderr.fileno()
fds = [ stdout_fileno, stderr_fileno ]
end_time = pscheduler.time_now() \
+ pscheduler.seconds_as_timedelta(timeout)
while True:
time_left = pscheduler.timedelta_as_seconds(
end_time - pscheduler.time_now() )
reads, writes, specials = select.select(fds, [], [], time_left)
if len(reads) == 0:
__running_drop(process)
return 2, None, "Process took too long to run."
for fd in reads:
if fd == stdout_fileno:
line = process.stdout.readline()
if line != '':
line_call(line[:-1])
elif fd == stderr_fileno:
line = process.stderr.readline()
if line != '':
stderr += line
if process.poll() != None:
break
process.wait()
status = process.returncode
stdout = None
except Exception as ex:
extype, ex_dummy, tb = sys.exc_info()
status = 2
stdout = ''
stderr = ''.join(traceback.format_exception_only(extype, ex)) \
+ ''.join(traceback.format_exception(extype, ex, tb)).strip()
if process is not None:
__running_drop(process)
if fail_message is not None and status != 0:
pscheduler.fail("%s: %s" % (fail_message, stderr))
return status, stdout, stderr