def __init__(
self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.source = data['source']
self.bind = data.get('bind', None)
self.update = pscheduler.iso8601_as_timedelta(data['update'])
self.retry = pscheduler.iso8601_as_timedelta(data['retry'])
self.fail_state = data.get('fail-state', False)
self.exclusions = radix.Radix()
if 'exclude' in data:
try:
for excl in data['exclude']:
self.exclusions.add(excl)
except ValueError:
raise ValueError("Invalid IP or CIDR '%s'" % excl)
# TODO: Would be nice to support a timeout so the system
# doesn't sit for too long.
self.cidrs = radix.Radix()
self.length = 0
# Prime the timer with the epoch and do a first load of the list
self.next_attempt = datetime.datetime.utcfromtimestamp(0)
self.__populate_cidrs__()
def __init__(self, drange):
"""Construct a range from a JSON DurationRange."""
# TODO: Would be nice if this class could treat missing
# lower/upper as zero or infinity.
# TODO: Figure out why this can't just point to a DurationRange
valid, message = pscheduler.json_validate(drange, {
"type": "object",
"properties": {
"lower": {"$ref": "#/pScheduler/Duration"},
"upper": {"$ref": "#/pScheduler/Duration"}
},
"additionalProperties": False,
"required": ["lower", "upper"]
})
if not valid:
raise ValueError("Invalid duration range: %s" % message)
self.lower_str = drange['lower']
self.lower = pscheduler.iso8601_as_timedelta(self.lower_str)
self.upper_str = drange['upper']
self.upper = pscheduler.iso8601_as_timedelta(self.upper_str)
def __init__(self, drange):
"""Construct a range from a JSON DurationRange."""
# TODO: Would be nice if this class could treat missing
# lower/upper as zero or infinity.
# TODO: Figure out why this can't just point to a DurationRange
valid, message = pscheduler.json_validate(drange, {
"type": "object",
"properties": {
"lower": { "$ref": "#/pScheduler/Duration" },
"upper": { "$ref": "#/pScheduler/Duration" }
},
"additionalProperties": False,
"required": [ "lower", "upper" ]
})
if not valid:
raise ValueError("Invalid duration range: %s" % message)
self.lower_str = drange['lower']
self.lower = pscheduler.iso8601_as_timedelta(self.lower_str)
self.upper_str = drange['upper']
self.upper = pscheduler.iso8601_as_timedelta(self.upper_str)
def __init__(
self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.exclude = [ipaddr.IPNetwork(addr) for addr in data['exclude']]
try:
timeout = pscheduler.iso8601_as_timedelta(data['timeout'])
self.timeout = pscheduler.timedelta_as_seconds(timeout)
except KeyError:
self.timeout = 2
try:
self.fail_result = data['fail-result']
except KeyError:
self.fail_result = False
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
def __init__(self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.exclude = [ ipaddr.IPNetwork(addr)
for addr in data['exclude'] ]
try:
timeout = pscheduler.iso8601_as_timedelta(data['timeout'])
self.timeout = pscheduler.timedelta_as_seconds(timeout)
except KeyError:
self.timeout = 2
try:
self.fail_result = data['fail-result']
except KeyError:
self.fail_result = False
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
def evaluate(
self,
proposal # Task and hints
):
"""Check that the proposed times don't overlap with this limit"""
start = pscheduler.iso8601_as_datetime(
proposal['task']['run_schedule']['start'])
duration = pscheduler.iso8601_as_timedelta(
proposal['task']['run_schedule']['duration'])
end = start + duration
subset = start >= self.start and end < self.end
if self.overlap:
passed = ((start <= self.start < end) or (start <= self.end < end)
or subset)
else:
passed = subset
result = {"passed": passed}
if not passed:
result['reasons'] = ["Ranges do not match"]
return result
def evaluate(self,
run # The proposed run
):
"""Check that the proposed times don't overlap with this limit"""
start = pscheduler.iso8601_as_datetime(run['schedule']['start'])
duration = pscheduler.iso8601_as_timedelta(run['schedule']['duration'])
end = start + duration
subset = start >= self.start and end < self.end
if self.overlap:
passed = ( (start <= self.start < end)
or (start <= self.end < end)
or subset
)
else:
passed = subset
result = { "passed": passed }
if not passed:
result['reasons'] = [ "Ranges do not match" ]
return result
def __init__(
self,
data # Data suitable for this class
):
valid, message = urlfetch_data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.url = data["url"]
self.url_transform = _jq_filter(data.get("url-transform", None))
self.bind = data.get("bind", None)
self.follow = data.get("follow-redirects", True)
self.timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(data.get("timeout", "PT3S")))
self.verify = data.get("verify-keys", True)
self.fail_result = data.get("fail-result", False)
self.headers = data.get("headers", {})
self.headers_transform = _jq_filter(data.get("headers-transform",
None))
self.params = data.get("params", {})
self.params_transform = _jq_filter(data.get("params-transform", None))
self.success_only = data.get("success-only", False)
def __init__(
self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.source = data['source']
self.bind = data.get('bind', None)
self.update = pscheduler.iso8601_as_timedelta(data['update'])
self.retry = pscheduler.iso8601_as_timedelta(data['retry'])
self.fail_state = data.get('fail-state', False)
try:
# This will raise a ValueError if it's wrong.
transform = data["transform"]
self.transform = pscheduler.JQFilter(transform["script"],
transform.get("args", {}),
output_raw=True)
except KeyError:
self.transform = None
self.exclusions = radix.Radix()
if 'exclude' in data:
try:
for excl in data['exclude']:
self.exclusions.add(excl)
except ValueError:
raise ValueError("Invalid IP or CIDR '%s'" % excl)
self.data_lock = threading.Lock()
self.updating = False
# TODO: Would be nice to support a timeout so the system
# doesn't sit for too long.
self.cidrs = radix.Radix()
self.length = 0
# Prime the timer with the epoch and do a first load of the list
self.next_attempt = datetime.datetime.utcfromtimestamp(0)
self.__populate_cidrs__()
def __contains__(self, duration):
"""See if the range contains the specified duration, which can be a
timedelta or ISO8601 string."""
if type(duration) == datetime.timedelta:
test_value = duration
elif type(duration) in [ str, unicode ]:
test_value = pscheduler.iso8601_as_timedelta(duration)
else:
raise ValueError("Invalid duration; must be ISO8601 string or timedelta.")
return self.lower <= test_value <= self.upper
def __contains__(self, duration):
"""See if the range contains the specified duration, which can be a
timedelta or ISO8601 string."""
if type(duration) == datetime.timedelta:
test_value = duration
elif type(duration) in [str, unicode]:
test_value = pscheduler.iso8601_as_timedelta(duration)
else:
raise ValueError(
"Invalid duration; must be ISO8601 string or timedelta.")
return self.lower <= test_value <= self.upper
def evaluate(
self,
proposal # Task and hints
):
"""Check that the proposed times don't overlap with this limit"""
start = pscheduler.iso8601_as_datetime(
proposal['task']['run_schedule']['start'])
duration = pscheduler.iso8601_as_timedelta(
proposal['task']['run_schedule']['duration'])
end = start + duration
# Python's datetime doesn't have methods to get this. Bravo.
start_week = start.isocalendar()[1]
end_week = end.isocalendar()[1]
match_failures = []
for name, lower, upper, wrap_after, wrap_to in [
# Feel free to resurrect me if this ever wraps. :-)
('year', start.year, end.year, 294276, 1),
('month', start.month, end.month, 12, 1),
('week', start_week, end_week, 53, 1),
('weekday', start.isoweekday(), end.isoweekday(), 7, 1),
('day', start.day, end.day, 31, 1),
('hour', start.hour, end.hour, 23, 0),
('minute', start.minute, end.minute, 59, 0),
('second', start.second, end.second, 59, 0)
]:
# Don't bother matching things that weren't specified
if name not in self.matches:
continue
if not wrappable_range_overlaps(lower,
upper,
self.matches[name],
wrap_after=wrap_after,
wrap_to=wrap_to,
overlap=self.overlap):
match_failures.append(name)
result = {"passed": not match_failures}
if match_failures:
result['reasons'] = [
"Mismatch on " + mis for mis in match_failures
]
return result
def __init__(self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.matcher = pscheduler.StringMatcher(data['match'])
timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(data['timeout']))
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = timeout
self.resolver.lifetime = timeout
def __init__(self,
data # Data suitable for this class
):
valid, message = data_is_valid(data)
if not valid:
raise ValueError("Invalid data: %s" % message)
self.matcher = pscheduler.StringMatcher(data['match'])
timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(data['timeout']))
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = timeout
self.resolver.lifetime = timeout
def evaluate(self,
run # The proposed run
):
"""Check that the proposed times don't overlap with this limit"""
start = pscheduler.iso8601_as_datetime(run['schedule']['start'])
duration = pscheduler.iso8601_as_timedelta(run['schedule']['duration'])
end = start + duration
# Python's datetime doesn't have methods to get this. Bravo.
start_week = start.isocalendar()[1]
end_week = end.isocalendar()[1]
match_failures = []
for name, lower, upper, wrap_after, wrap_to in [
# Feel free to resurrect me if this ever wraps. :-)
('year', start.year, end.year, 294276, 1),
('month', start.month, end.month, 12, 1),
('week', start_week, end_week, 53, 1),
('weekday', start.isoweekday(), end.isoweekday(), 7, 1),
('day', start.day, end.day, 31, 1),
('hour', start.hour, end.hour, 23, 0),
('minute', start.minute, end.minute, 59, 0),
('second', start.second, end.second, 59, 0)
]:
# Don't bother matching things that weren't specified
if name not in self.matches:
continue
if not wrappable_range_overlaps(lower, upper, self.matches[name],
wrap_after=wrap_after,
wrap_to=wrap_to,
overlap=self.overlap):
match_failures.append(name)
result = { "passed": not match_failures }
if match_failures:
result['reasons'] = [ "Mismatch on " + mis
for mis in match_failures ]
return result
def setup_time(rtt):
"""Setup time: How long it takes for iperf2 to set up a test. This is
heavily influenced by the RTT between the hosts involved.
Measurements taken as part of #868 indicate that every 50 ms of
latency adds about one second. Previously, this was done with a
static fudge factor of four seconds, which has been set as the
default. RTT has no influence on how long it takes to run the
test; iperf2 will stop after the specified number of seconds
regardless of how much data was transferred.
The rtt is an ISO8601 duration as a string. If None, the default
will be used.
Returned value is seconds, minimum 1.
"""
delta = pscheduler.iso8601_as_timedelta(rtt or DEFAULT_LINK_RTT)
rtt = pscheduler.timedelta_as_seconds(delta)
# TODO: This is the same as what we do for iperf3 but may need tuning.
return max(((rtt * 1000.0) / 50.0), 1.0)
def iso8601_to_seconds(val):
td = pscheduler.iso8601_as_timedelta(val)
return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10.0**6) / 10.0**6
def run(input):
# TODO: Check the spec schema
try:
assert input["test"]["type"] == "http"
source = input['test']['spec']['url']
always_succeed = input['test']['spec'].get('always-succeed', False)
keep_content = input['test']['spec'].get('keep-content', None)
timeout_iso = input['test']['spec'].get('timeout', 'PT10S')
ip_version = input['test']['spec'].get('ip-version', None)
timeout = pscheduler.timedelta_as_seconds(
pscheduler.iso8601_as_timedelta(timeout_iso))
except KeyError as ex:
return ({"succeeded": False, "error": "Missing data in input"})
# Can-run should weed these out, but backstop it with a check just in case.
if source[0:5] == "file:" and keep_content is not None:
return ({
"succeeded": False,
"error": "Cannot keep content from file:// URLs",
"diags": None
})
succeeded = False
error = None
diags = []
STDERR = ""
# TODO: Implement this with libcurl
curl = pycurl.Curl()
curl.setopt(pycurl.URL, str(source))
# TODO: This test doesn't have bind but needs one.
# curl.setopt(pycurl.INTERFACE, str(bind))
# TODO: Redirects as an option?
# curl.setopt(pycurl.FOLLOWLOCATION, allow_redirects)
if timeout is not None:
curl.setopt(pycurl.TIMEOUT_MS, int(timeout * 1000.0))
curl.setopt(pycurl.SSL_VERIFYHOST, False)
curl.setopt(pycurl.SSL_VERIFYPEER, False)
if ip_version is not None:
curl.setopt(
pycurl.IPRESOLVE,
pycurl.IPRESOLVE_V4 if ip_version == 4 else pycurl.IPRESOLVE_V6)
buf = io.BytesIO()
curl.setopt(pycurl.WRITEFUNCTION, buf.write)
text = ""
try:
start_time = datetime.datetime.now()
curl.perform()
status = curl.getinfo(pycurl.HTTP_CODE)
# PycURL returns a zero for non-HTTP URLs
if status == 0:
status = 200
text = buf.getvalue().decode()
except pycurl.error as ex:
code, message = ex.args
status = 400
text = message
finally:
end_time = datetime.datetime.now()
curl.close()
buf.close()
# 200-299 is success; anything else is an error.
fetch_succeeded = (status >= 200 and status < 300)
succeeded = always_succeed or fetch_succeeded
if succeeded:
schema = pscheduler.HighInteger(1)
run_result = {
"succeeded": True,
"time": pscheduler.timedelta_as_iso8601(end_time - start_time)
}
if always_succeed:
run_result["status"] = status
schema.set(2)
try:
run_result["found"] = text.find(
input['test']['spec']["parse"]) >= 0
except KeyError:
pass
# If the fetch failed or we've been told to keep 0 content, plaster it all in.
if (not fetch_succeeded) or (keep_content is not None
and keep_content == 0):
run_result["content"] = text
schema.set(2)
elif keep_content is not None:
run_result["content"] = text[:keep_content]
schema.set(2)
run_result["schema"] = schema.value()
return {
"succeeded": True,
"diags": None,
"error": None,
"result": run_result
}
else:
return {
"succeeded": False,
"diags": "Fetch returned non-success status %d" % (status),
"error": text
}
assert False, "Should not be reached."
def iso8601_to_seconds(val):
td = pscheduler.iso8601_as_timedelta(val)
return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10.0**6) / 10.0**6
def iso8601_to_seconds(val):
return pscheduler.iso8601_as_timedelta(val).total_seconds()
