def __init__( self, data # Data suitable for this class ): valid, message = data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.source = data['source'] self.bind = data.get('bind', None) self.update = pscheduler.iso8601_as_timedelta(data['update']) self.retry = pscheduler.iso8601_as_timedelta(data['retry']) self.fail_state = data.get('fail-state', False) self.exclusions = radix.Radix() if 'exclude' in data: try: for excl in data['exclude']: self.exclusions.add(excl) except ValueError: raise ValueError("Invalid IP or CIDR '%s'" % excl) # TODO: Would be nice to support a timeout so the system # doesn't sit for too long. self.cidrs = radix.Radix() self.length = 0 # Prime the timer with the epoch and do a first load of the list self.next_attempt = datetime.datetime.utcfromtimestamp(0) self.__populate_cidrs__()
def __init__(self, drange): """Construct a range from a JSON DurationRange.""" # TODO: Would be nice if this class could treat missing # lower/upper as zero or infinity. # TODO: Figure out why this can't just point to a DurationRange valid, message = pscheduler.json_validate(drange, { "type": "object", "properties": { "lower": {"$ref": "#/pScheduler/Duration"}, "upper": {"$ref": "#/pScheduler/Duration"} }, "additionalProperties": False, "required": ["lower", "upper"] }) if not valid: raise ValueError("Invalid duration range: %s" % message) self.lower_str = drange['lower'] self.lower = pscheduler.iso8601_as_timedelta(self.lower_str) self.upper_str = drange['upper'] self.upper = pscheduler.iso8601_as_timedelta(self.upper_str)
def __init__(self, drange): """Construct a range from a JSON DurationRange.""" # TODO: Would be nice if this class could treat missing # lower/upper as zero or infinity. # TODO: Figure out why this can't just point to a DurationRange valid, message = pscheduler.json_validate(drange, { "type": "object", "properties": { "lower": { "$ref": "#/pScheduler/Duration" }, "upper": { "$ref": "#/pScheduler/Duration" } }, "additionalProperties": False, "required": [ "lower", "upper" ] }) if not valid: raise ValueError("Invalid duration range: %s" % message) self.lower_str = drange['lower'] self.lower = pscheduler.iso8601_as_timedelta(self.lower_str) self.upper_str = drange['upper'] self.upper = pscheduler.iso8601_as_timedelta(self.upper_str)
def __init__( self, data # Data suitable for this class ): valid, message = data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.exclude = [ipaddr.IPNetwork(addr) for addr in data['exclude']] try: timeout = pscheduler.iso8601_as_timedelta(data['timeout']) self.timeout = pscheduler.timedelta_as_seconds(timeout) except KeyError: self.timeout = 2 try: self.fail_result = data['fail-result'] except KeyError: self.fail_result = False self.resolver = dns.resolver.Resolver() self.resolver.timeout = self.timeout self.resolver.lifetime = self.timeout
def __init__(self, data # Data suitable for this class ): valid, message = data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.exclude = [ ipaddr.IPNetwork(addr) for addr in data['exclude'] ] try: timeout = pscheduler.iso8601_as_timedelta(data['timeout']) self.timeout = pscheduler.timedelta_as_seconds(timeout) except KeyError: self.timeout = 2 try: self.fail_result = data['fail-result'] except KeyError: self.fail_result = False self.resolver = dns.resolver.Resolver() self.resolver.timeout = self.timeout self.resolver.lifetime = self.timeout
def evaluate( self, proposal # Task and hints ): """Check that the proposed times don't overlap with this limit""" start = pscheduler.iso8601_as_datetime( proposal['task']['run_schedule']['start']) duration = pscheduler.iso8601_as_timedelta( proposal['task']['run_schedule']['duration']) end = start + duration subset = start >= self.start and end < self.end if self.overlap: passed = ((start <= self.start < end) or (start <= self.end < end) or subset) else: passed = subset result = {"passed": passed} if not passed: result['reasons'] = ["Ranges do not match"] return result
def evaluate(self, run # The proposed run ): """Check that the proposed times don't overlap with this limit""" start = pscheduler.iso8601_as_datetime(run['schedule']['start']) duration = pscheduler.iso8601_as_timedelta(run['schedule']['duration']) end = start + duration subset = start >= self.start and end < self.end if self.overlap: passed = ( (start <= self.start < end) or (start <= self.end < end) or subset ) else: passed = subset result = { "passed": passed } if not passed: result['reasons'] = [ "Ranges do not match" ] return result
def __init__( self, data # Data suitable for this class ): valid, message = urlfetch_data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.url = data["url"] self.url_transform = _jq_filter(data.get("url-transform", None)) self.bind = data.get("bind", None) self.follow = data.get("follow-redirects", True) self.timeout = pscheduler.timedelta_as_seconds( pscheduler.iso8601_as_timedelta(data.get("timeout", "PT3S"))) self.verify = data.get("verify-keys", True) self.fail_result = data.get("fail-result", False) self.headers = data.get("headers", {}) self.headers_transform = _jq_filter(data.get("headers-transform", None)) self.params = data.get("params", {}) self.params_transform = _jq_filter(data.get("params-transform", None)) self.success_only = data.get("success-only", False)
def __init__( self, data # Data suitable for this class ): valid, message = data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.source = data['source'] self.bind = data.get('bind', None) self.update = pscheduler.iso8601_as_timedelta(data['update']) self.retry = pscheduler.iso8601_as_timedelta(data['retry']) self.fail_state = data.get('fail-state', False) try: # This will raise a ValueError if it's wrong. transform = data["transform"] self.transform = pscheduler.JQFilter(transform["script"], transform.get("args", {}), output_raw=True) except KeyError: self.transform = None self.exclusions = radix.Radix() if 'exclude' in data: try: for excl in data['exclude']: self.exclusions.add(excl) except ValueError: raise ValueError("Invalid IP or CIDR '%s'" % excl) self.data_lock = threading.Lock() self.updating = False # TODO: Would be nice to support a timeout so the system # doesn't sit for too long. self.cidrs = radix.Radix() self.length = 0 # Prime the timer with the epoch and do a first load of the list self.next_attempt = datetime.datetime.utcfromtimestamp(0) self.__populate_cidrs__()
def __contains__(self, duration): """See if the range contains the specified duration, which can be a timedelta or ISO8601 string.""" if type(duration) == datetime.timedelta: test_value = duration elif type(duration) in [ str, unicode ]: test_value = pscheduler.iso8601_as_timedelta(duration) else: raise ValueError("Invalid duration; must be ISO8601 string or timedelta.") return self.lower <= test_value <= self.upper
def __contains__(self, duration): """See if the range contains the specified duration, which can be a timedelta or ISO8601 string.""" if type(duration) == datetime.timedelta: test_value = duration elif type(duration) in [str, unicode]: test_value = pscheduler.iso8601_as_timedelta(duration) else: raise ValueError( "Invalid duration; must be ISO8601 string or timedelta.") return self.lower <= test_value <= self.upper
def evaluate( self, proposal # Task and hints ): """Check that the proposed times don't overlap with this limit""" start = pscheduler.iso8601_as_datetime( proposal['task']['run_schedule']['start']) duration = pscheduler.iso8601_as_timedelta( proposal['task']['run_schedule']['duration']) end = start + duration # Python's datetime doesn't have methods to get this. Bravo. start_week = start.isocalendar()[1] end_week = end.isocalendar()[1] match_failures = [] for name, lower, upper, wrap_after, wrap_to in [ # Feel free to resurrect me if this ever wraps. :-) ('year', start.year, end.year, 294276, 1), ('month', start.month, end.month, 12, 1), ('week', start_week, end_week, 53, 1), ('weekday', start.isoweekday(), end.isoweekday(), 7, 1), ('day', start.day, end.day, 31, 1), ('hour', start.hour, end.hour, 23, 0), ('minute', start.minute, end.minute, 59, 0), ('second', start.second, end.second, 59, 0) ]: # Don't bother matching things that weren't specified if name not in self.matches: continue if not wrappable_range_overlaps(lower, upper, self.matches[name], wrap_after=wrap_after, wrap_to=wrap_to, overlap=self.overlap): match_failures.append(name) result = {"passed": not match_failures} if match_failures: result['reasons'] = [ "Mismatch on " + mis for mis in match_failures ] return result
def __init__(self, data # Data suitable for this class ): valid, message = data_is_valid(data) if not valid: raise ValueError("Invalid data: %s" % message) self.matcher = pscheduler.StringMatcher(data['match']) timeout = pscheduler.timedelta_as_seconds( pscheduler.iso8601_as_timedelta(data['timeout'])) self.resolver = dns.resolver.Resolver() self.resolver.timeout = timeout self.resolver.lifetime = timeout
def evaluate(self, run # The proposed run ): """Check that the proposed times don't overlap with this limit""" start = pscheduler.iso8601_as_datetime(run['schedule']['start']) duration = pscheduler.iso8601_as_timedelta(run['schedule']['duration']) end = start + duration # Python's datetime doesn't have methods to get this. Bravo. start_week = start.isocalendar()[1] end_week = end.isocalendar()[1] match_failures = [] for name, lower, upper, wrap_after, wrap_to in [ # Feel free to resurrect me if this ever wraps. :-) ('year', start.year, end.year, 294276, 1), ('month', start.month, end.month, 12, 1), ('week', start_week, end_week, 53, 1), ('weekday', start.isoweekday(), end.isoweekday(), 7, 1), ('day', start.day, end.day, 31, 1), ('hour', start.hour, end.hour, 23, 0), ('minute', start.minute, end.minute, 59, 0), ('second', start.second, end.second, 59, 0) ]: # Don't bother matching things that weren't specified if name not in self.matches: continue if not wrappable_range_overlaps(lower, upper, self.matches[name], wrap_after=wrap_after, wrap_to=wrap_to, overlap=self.overlap): match_failures.append(name) result = { "passed": not match_failures } if match_failures: result['reasons'] = [ "Mismatch on " + mis for mis in match_failures ] return result
def setup_time(rtt): """Setup time: How long it takes for iperf2 to set up a test. This is heavily influenced by the RTT between the hosts involved. Measurements taken as part of #868 indicate that every 50 ms of latency adds about one second. Previously, this was done with a static fudge factor of four seconds, which has been set as the default. RTT has no influence on how long it takes to run the test; iperf2 will stop after the specified number of seconds regardless of how much data was transferred. The rtt is an ISO8601 duration as a string. If None, the default will be used. Returned value is seconds, minimum 1. """ delta = pscheduler.iso8601_as_timedelta(rtt or DEFAULT_LINK_RTT) rtt = pscheduler.timedelta_as_seconds(delta) # TODO: This is the same as what we do for iperf3 but may need tuning. return max(((rtt * 1000.0) / 50.0), 1.0)
def iso8601_to_seconds(val): td = pscheduler.iso8601_as_timedelta(val) return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10.0**6) / 10.0**6
def run(input): # TODO: Check the spec schema try: assert input["test"]["type"] == "http" source = input['test']['spec']['url'] always_succeed = input['test']['spec'].get('always-succeed', False) keep_content = input['test']['spec'].get('keep-content', None) timeout_iso = input['test']['spec'].get('timeout', 'PT10S') ip_version = input['test']['spec'].get('ip-version', None) timeout = pscheduler.timedelta_as_seconds( pscheduler.iso8601_as_timedelta(timeout_iso)) except KeyError as ex: return ({"succeeded": False, "error": "Missing data in input"}) # Can-run should weed these out, but backstop it with a check just in case. if source[0:5] == "file:" and keep_content is not None: return ({ "succeeded": False, "error": "Cannot keep content from file:// URLs", "diags": None }) succeeded = False error = None diags = [] STDERR = "" # TODO: Implement this with libcurl curl = pycurl.Curl() curl.setopt(pycurl.URL, str(source)) # TODO: This test doesn't have bind but needs one. # curl.setopt(pycurl.INTERFACE, str(bind)) # TODO: Redirects as an option? # curl.setopt(pycurl.FOLLOWLOCATION, allow_redirects) if timeout is not None: curl.setopt(pycurl.TIMEOUT_MS, int(timeout * 1000.0)) curl.setopt(pycurl.SSL_VERIFYHOST, False) curl.setopt(pycurl.SSL_VERIFYPEER, False) if ip_version is not None: curl.setopt( pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4 if ip_version == 4 else pycurl.IPRESOLVE_V6) buf = io.BytesIO() curl.setopt(pycurl.WRITEFUNCTION, buf.write) text = "" try: start_time = datetime.datetime.now() curl.perform() status = curl.getinfo(pycurl.HTTP_CODE) # PycURL returns a zero for non-HTTP URLs if status == 0: status = 200 text = buf.getvalue().decode() except pycurl.error as ex: code, message = ex.args status = 400 text = message finally: end_time = datetime.datetime.now() curl.close() buf.close() # 200-299 is success; anything else is an error. fetch_succeeded = (status >= 200 and status < 300) succeeded = always_succeed or fetch_succeeded if succeeded: schema = pscheduler.HighInteger(1) run_result = { "succeeded": True, "time": pscheduler.timedelta_as_iso8601(end_time - start_time) } if always_succeed: run_result["status"] = status schema.set(2) try: run_result["found"] = text.find( input['test']['spec']["parse"]) >= 0 except KeyError: pass # If the fetch failed or we've been told to keep 0 content, plaster it all in. if (not fetch_succeeded) or (keep_content is not None and keep_content == 0): run_result["content"] = text schema.set(2) elif keep_content is not None: run_result["content"] = text[:keep_content] schema.set(2) run_result["schema"] = schema.value() return { "succeeded": True, "diags": None, "error": None, "result": run_result } else: return { "succeeded": False, "diags": "Fetch returned non-success status %d" % (status), "error": text } assert False, "Should not be reached."
def iso8601_to_seconds(val): return pscheduler.iso8601_as_timedelta(val).total_seconds()