def parse(cls, value): return cls(**valideer.parse({ "+a": valideer.Boolean, "+b": valideer.Boolean, "+c": valideer.Boolean, "+d": valideer.AdaptTo(float), "+e": valideer.AdaptTo(int), "+f": valideer.AdaptTo(int), }).validate(value))
def test_adapt_to(self): self.assertRaises(TypeError, V.AdaptTo, hex) for exact in False, True: self._testValidation(V.AdaptTo(int, traps=(ValueError, TypeError), exact=exact), invalid=["12b", "1.2", {}, (), []], adapted=[(12, 12), ("12", 12), (1.2, 1)]) class smallint(int): pass i = smallint(2) self.assertIs(V.AdaptTo(int).validate(i), i) self.assertIsNot(V.AdaptTo(int, exact=True).validate(i), i)
def __init__(self, allow_extra): schema = { '+id': int, '+client_name': V.String(max_length=255), '+sort_index': float, 'client_phone': V.Nullable(V.String(max_length=255)), 'location': {'latitude': float, 'longitude': float}, 'contractor': V.Range(V.AdaptTo(int), min_value=1), 'upstream_http_referrer': V.Nullable(V.String(max_length=1023)), '+grecaptcha_response': V.String(min_length=20, max_length=1000), 'last_updated': V.AdaptBy(dateutil.parser.parse), 'skills': V.Nullable( [ { '+subject': str, '+subject_id': int, '+category': str, '+qual_level': str, '+qual_level_id': int, 'qual_level_ranking': V.Nullable(float, default=0), } ], default=[], ), } self.validator = V.parse(schema, additional_properties=allow_extra)
def test_register(self): for register in V.register, V.Validator.register: register("to_int", V.AdaptTo(int, traps=(ValueError, TypeError))) self._testValidation("to_int", invalid=["12b", "1.2"], adapted=[(12, 12), ("12", 12), (1.2, 1)]) self.assertRaises(TypeError, register, "to_int", int)
def test_chainof(self): self._testValidation(V.ChainOf(V.AdaptTo(int), V.Condition(lambda x: x > 0), V.AdaptBy(datetime.utcfromtimestamp)), adapted=[(1373475820, datetime(2013, 7, 10, 17, 3, 40)), ("1373475820", datetime(2013, 7, 10, 17, 3, 40))], invalid=["nan", -1373475820])
def info(env): schema = v.parse({'offset': v.AdaptTo(int)}) args = schema.validate(env.request.args) if args.get('offset'): env.session['tz_offset'] = args['offset'] if env.valid_username or env.valid_token: res = ctx_init(env) else: res = {} return res
def __checkSanitarinessOfPayloadMeta(self, dictPayload): # check that bad guys haven't put dangerous content into the input # we can't trap malicious output here because it's trivial to change this program to not do the check try: # dictPayload is a python dictionary # check the content is the right format for the schema # adapt_sent = V.parse({"+snt" : [V.AdaptTo(String)]}) payload_schema = { "+snt" : [V.AdaptTo(String)], "+sz" : [V.AdaptTo(Integer)], "+szu" : [V.AdaptTo(Integer)], "+szc" : [V.AdaptTo(Integer)], "+sze" : [V.AdaptTo(Integer)], "+cks" : [V.AdaptTo(String)], "+dat" : "string", ### should be in uuencode format i.e. alphanumerics } # payload_schema = { # V.parse({"+snt" : [V.AdaptTo(String)]}), # V.parse({"+sz" : [V.AdaptTo(Integer)]}), # V.parse({"+szu" : [V.AdaptTo(Integer)]}), # V.parse({"+szc" : [V.AdaptTo(Integer)]}), # V.parse({"+sze" : [V.AdaptTo(Integer)]}), # V.parse({"+cks" : [V.AdaptTo(String)]}), # "+dat" : "string", ### should be in uuencode format i.e. alphanumerics # } # payload_schema = { # "+snt": "datetime", # "+sz": V.Range("number", min_value=0), # "+szu": V.Range("number", min_value=0), # "+szc": V.Range("number", min_value=0), # "+sze": V.Range("number", min_value=0), # "+cks": "string", # "+dat" : "string", ### should be in uuencode format i.e. alphanumerics # } validator = V.parse(payload_schema) # dictValidatePayload = { # "snt": int(unicodeQos.decode('utf-8')), # "sz": , # "szu": , # "szc": , # "sze": , # "cks": , # "dat" : , ### should be in uuencode format i.e. alphanumerics # # } # (int(unicodeQos.decode('utf-8'))) validator.validate(dictPayload) except V.ValidationError as e: logging.error("ERROR - invalid data format in dictionary to send - %s", e) return False except: logging.error("ERROR - invalid data format in dictionary to send - %s", sys.exc_info()[0]) return False return True
def inner(env, *a, **kw): schema = v.parse({ 'page': v.Nullable(v.AdaptTo(int), 1), 'last': v.Nullable(str) }) data = schema.validate(env.request.args) page, last = data['page'], data.get('last') page = { 'limit': env('ui_per_page'), 'offset': env('ui_per_page') * (page - 1), 'last': last, 'count': env('ui_per_page') * page, 'current': page, 'next': page + 1, } return wrapper.func(env, page, *a, **kw)
class StatusMessage(BaseMessage): hostname = attr.ib() load = attr.ib() projects = attr.ib() pid = attr.ib() _validator = V.parse({ "hostname": "string", "load": V.AdaptTo(tuple, ("number", "number", "number")), "pid": "integer", "projects": V.Mapping( "string", { "git": "string", "description": "string", "classes": ["string"], "running": "boolean", }) })
class Heartbeat(BaseMessage): # status and statistics load = attr.ib() fqdn = attr.ib() # These are for display purposes on the web interface project = attr.ib(default=None) job = attr.ib(default=None) @staticmethod def _filter(x, v): return v is not None _validator = V.parse({ "load": V.AdaptTo(tuple, ("number", "number", "number")), "fqdn": "string", "?project": "string", "?job": "string" })
def __checkSanitarinessOfID(self, dictMetaHdr): try: # dictMetaHdr is a python dictionary # check the content is the right format for the schema header_schema = { "+ver": [V.AdaptTo(String)], "+sndr": [V.AdaptTo(String)], "+name": [V.AdaptTo(String)], # "+ver": "string", # "+sndr": "string", # "+name": "string", } # header_schema = { # V.parse({"+ver" : [V.AdaptTo(String)]}), # V.parse({"+sndr" : [V.AdaptTo(String)]}), # V.parse({"+name" : [V.AdaptTo(String)]}), ## "+ver": "string", ## "+sndr": "string", ## "+name": "string", # } validator = V.parse(header_schema) if validator.is_valid(dictMetaHdr): pass else: logging.error("ERROR - schema layout incorrect in send header") return False except V.ValidationError as e: logging.error( "ERROR - schema layout incorrect in send header - %s", e) return False except TypeError as e: logging.error( "ERROR - schema layout incorrect in send header - typeerror %s", e) return False except ValueError as e: logging.error( "ERROR - schema layout incorrect in send header - valueerror %s", e) return False except AttributeError as e: logging.error( "ERROR - schema layout incorrect in send header - attributeerror %s", e) return False except: logging.error( "ERROR - schema layout incorrect in send header - error is - %s", sys.exc_info()[0]) return False # cant get this to work right so leaving it out for now # try: # try: # tempMatch = unicodedata.normalize('NFC',friendlyName) # except: # pass # # if re.match(r'^[^\w-]+$', str(tempMatch)) is not None: # check it is only alphanumerics and underscore and hyphen from start to end # pass # else: # logging.error("ERROR - friendlyName must be alphanumeric, hyphen or underscore in send") # return False # except TypeError as e: # logging.error("ERROR - friendlyName must be alphanumeric, hyphen or underscore in send - typeerror %s", e) # return False # except ValueError as e: # logging.error("ERROR - friendlyName must be alphanumeric, hyphen or underscore in send - valueerror %s", e) # return False # except AttributeError as e: # logging.error("ERROR - friendlyName must be alphanumeric, hyphen or underscore in send - attributeerror %s", e) # return False # except ValueError as e: # logging.error("ERROR - friendlyName must be alphanumeric, hyphen or underscore in send - valueerror %s", e) # return False # except: # logging.error("ERROR - friendlyName must be alphanumeric, hyphen or underscore in send - error is - %s", sys.exc_info()[0]) # return False return True
class TestSuiteCitySDKKlarschiff(citysdk311.TestSuiteCitySDK): SCHEMA_REQUESTSEXT = { "+service_requests": { "+request": [{ "extended_attributes": { "title": "string", #TOOD: are EXT attributes optional even if requested? "service_object_id": "string", "service_object_type": "string", "detailed_status": valideer.Enum([ "RECEIVED", "IN_PROCESS", "PROCESSED", "ARCHIVED", "REJECTED" ]), "detailed_status_datetime": valideer.AdaptBy(parser.parse), "media_urls": { "media_url": ["string"] } }, "photo_required": "boolean", "trust": "integer", "votes": "integer", "+service_request_id": "string", "+status": valideer.Enum(["OPEN", "CLOSED"]), "status_notes": "string", "+service_name": "string", "+service_code": "string", "description": "string", "agency_responsible": "string", "service_notice": "string", "requested_datetime": valideer.AdaptBy(parser.parse), "updated_datetime": valideer.AdaptBy(parser.parse), "expected_datetime": valideer.AdaptBy(parser.parse), "address": "string", #TODO: Make Position XOR address "address_id": "string", "zipcode": "string", "+lat": valideer.AdaptTo(float, "number"), "+long": valideer.AdaptTo(float, "number"), "media_url": "string", }] #TODO: make URL regex } } SCHEMA_COMMENTS = { "+comments": { #TODO: Conditional Array? "+comment": { "+id": "integer", "+jurisdiction_id": "string", "+comment": "string", "+datetime": valideer.AdaptBy(parser.parse), "+service_request_id": "string", "+author": "string", #TODO: Validate as email } } } SCHEMA_NOTES = { "+notes": { #TODO: Conditional Array? "+note": { "+jurisdiction_id": "string", "+comment": "string", "+datetime": valideer.AdaptBy(parser.parse), "+service_request_id": "string", "+author": "string" } } } def __init__(self, host): self.api = CitysdkKlarschiff(host) citysdk311.TestSuiteCitySDK.__init__(self, host, self.api) def testRequestsExtKS(self): print("Testing GET extended requests"), repl = self.api.getRequests() expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") replFields = self.xmlToDict(repl.content) print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_REQUESTSEXT) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) def testGetRequestsExFilteredKS(self, agency): print("Testing GET requests with extended filters") print("-agency"), repl = self.api.getRequests(lat=12.13955, long=54.09138, radius=2000) #TODO calculate that point self.testEmptyRequestSet(repl) print("-detailed_status"), repl = self.api.getRequests(statusEx="RECEIVED") self.testEmptyRequestSet(repl) def testGetComments(self, apikey): repl = self.api.getComments(66, apikey) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") replFields = self.xmlToDict(repl.content) print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_COMMENTS) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) def testGetNotes(self, apikey): repl = self.api.getNotes(66, apikey) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") replFields = self.xmlToDict(repl.content) print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_NOTES) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e)))
class TestSuite(object): '''Open311 basic test. Inherited classes just run it's own tests''' SCHEMA_SERVICE = { "+services": { "+service": [{ "+service_code": "string", "+service_name": "string", "description": "string", "+metadata": valideer.Enum(["true", "false"]), "+type": valideer.Enum(["realtime", "batch", "blackbox"]), "+keywords": "string", "group": "string", }] } } SCHEMA_DEF = { "+service_code": "string", "+attributes": [{ "+variable": valideer.Enum(["true", "false"]), "+code": "string", "+datatype": valideer.Enum([ "string", "number", "datetime", "text", "singlevaluelist", "multivaluelist" ]), "+required": valideer.Enum(["true", "false"]), "+datatype_description": "string", "+order": valideer.Range(valideer.Number, 0), "description": "string" }] } SCHEMA_REQUESTS = { "+service_requests": { "+request": [{ "+service_request_id": "string", "+status": valideer.Enum(["open", "closed"]), "status_notes": "string", "+service_name": "string", "+service_code": "string", "description": "string", "agency_responsible": "string", "service_notice": "string", "requested_datetime": valideer.AdaptBy(parser.parse), "updated_datetime": valideer.AdaptBy(parser.parse), "expected_datetime": valideer.AdaptBy(parser.parse), "address": "string", #TODO: Make Position XOR address "address_id": "string", "zipcode": "string", "+lat": valideer.AdaptTo(float, "number"), "+long": valideer.AdaptTo(float, "number"), "media_url": "string", }] #TODO: make URL regex } } SCHEMA_REQUEST = { "+service_requests": { #TODO: Try to remove slightly redundant schema "+request": { "+service_request_id": "string", "+status": valideer.Enum(["open", "closed"]), "status_notes": "string", "+service_name": "string", "+service_code": "string", "description": "string", "agency_responsible": "string", "service_notice": "string", "requested_datetime": valideer.AdaptBy(parser.parse), "updated_datetime": valideer.AdaptBy(parser.parse), "expected_datetime": valideer.AdaptBy(parser.parse), "address": "string", #TODO: Make Position XOR address "address_id": "string", "+zipcode": "string", "+lat": valideer.AdaptTo(float, "number"), "+long": valideer.AdaptTo(float, "number"), "media_url": "string", } #TODO: make URL regex } } @staticmethod def __noNone(path, key, value): '''we skip dict none values for like <key/> elements ''' if value is None: return key, "" '''we skip additional typisation fields''' if (isinstance(value, dict)): if "@nil" in value: return key, "" if "@type" in value: if "#text" in value: return key, value["#text"] #if value.hasKey("@type"): oder alle mit @ return key, value def __init__(self, host, api=None, apikey=None): self.host = host if api is None: api = georeporter(host, apikey) self.api = api self.__cacheTransfers() def showDifferences(self, expect, result, msgTest): '''helper for visual compare of two lists''' #TODO: Add normalizing option - lowercase, trim, ... s = set(expect) diff = [x for x in result if x not in s] print("- " + msgTest), if len(diff) == 0: print("OK") else: print("differs at: %s (expected: %s)" % (str(diff), str(expect))) def __cacheTransfers(self): '''call api just one time for defaults''' print "caching..." print "-", self.cache_services = self.api.getServices() print "-", self.cache_requests = self.api.getRequests() #TODO: Write to disk? def __getFirstServiceCode(self): serviceCode = None root = etree.fromstring(self.cache_services.content) for service in root.getchildren(): fields = self.xmlToDict(etree.tostring(service)) try: if fields["metadata"] == "true": serviceCode = fields["service_code"] return serviceCode except: pass #TODO: Log that metadata is mandatory? return serviceCode def __getFirstRequestId(self): srid = None root = etree.fromstring(self.cache_requests.content) for servicereq in root.getchildren(): fields = self.xmlToDict(etree.tostring(servicereq))["request"] if "service_request_id" in fields: srid = fields["service_request_id"] return srid return srid def xmlToDict(self, repl): '''Transform XML tree to dict tree''' tmp = xmltodict.parse(repl, postprocessor=self.__noNone) replFields = json.loads(json.dumps(tmp)) return replFields #TODO: Test discovery def testGetServices(self): print("Testing GET services") repl = self.cache_services expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_SERVICE) replFields = self.xmlToDict(repl.content) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs %s " % (str(e))) def testGetServicsDef(self): #TODO: walk trough all definitions print("Testing GET service definition"), firstCode = self.__getFirstServiceCode() if firstCode is not None: repl = self.api.getServiceDef(firstCode) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [ repl.status_code, repl.encoding, repl.headers["content-type"] ] self.showDifferences(expect, resp, "http skelleton") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_DEF) replFields = self.xmlToDict(repl) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) else: print "(No service definitions available for testing)" def testCreateRequest(self, email="*****@*****.**", lat=54.0867, lon=12.1359, descr="test", title="test", code="18"): #TODO: Extract attributes print("Testing POST request"), repl = self.api.createRequest(lat, lon, code, email, descr, attr={"title": title}, jurisdinction="rostock.de", tolerantSSL=True) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") #TODO: Attributes of metadata / service definitions #TODO: what more can we check here? #TODO: Some might return token instead def testGetRequests(self): print("Testing GET requests"), repl = self.cache_requests expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_REQUESTS) replFields = self.xmlToDict(repl.content) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) def testGetRequestsFiltered(self): print("Testing GET requests with filters") #TODO: How to generate indipendend queries? #TODO: How to check logic on expected results? print("-service_request_id"), repl = self.api.getRequests(service_request_id="3,5") self.testEmptyRequestSet(repl) print("-service_code") repl = self.api.getRequests(service_code=18) self.testEmptyRequestSet(repl) print("-start_date") past = datetime.datetime.utcnow() - datetime.timedelta(2) repl = self.api.getRequests(start_date=past.isoformat()) self.testEmptyRequestSet(repl) print("-end_date") repl = self.api.getRequests(end_date=past.isoformat()) self.testEmptyRequestSet(repl) print("-status") repl = self.api.getRequests(status="closed") self.testEmptyRequestSet(repl) #TODO: None older than 90days? def testGetRequest(self): print("Testing GET request"), srid = self.__getFirstRequestId() repl = self.api.getRequest(srid) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") print("- XML structure") with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_REQUEST) replFields = self.xmlToDict(repl.content) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) def testGetRequestFromToken(self): print("Testing GET ID by token"), mytoken = "123" #TODO: Implement lookup for an valid token? repl = self.api.getIdByToken(mytoken) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") def testEmptyRequestSet(self, repl): expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") print "- queryset", replFields = self.xmlToDict(repl.content) amount = 1 #if no subdicts, just a single request try: for item in replFields["service_requests"]["request"]: if isinstance(item, dict): amount += 1 if (amount > 0): print "ok (%d)" % amount else: print("failed (empty)") except KeyError: print("failed (empty)")
class TestSuiteCitySDK(basic311.TestSuite): '''Cover CitySDK specific tests''' SCHEMA_REQUESTSEXT = { "+service_requests": { "+request": [{ "extended_attributes": { "title": "string", #TOOD: are EXT attributes optional even if requested? "service_object_id": "string", "service_object_type": "string", "detailed_status": valideer.Enum([ "RECEIVED", "IN_PROCESS", "PROCESSED", "ARCHIVED", "REJECTED" ]), #TODO: Add Public works status "media_urls": { "media_url": ["string"] } } #TODO: some implement , "+service_request_id": "string", "+status": valideer.Enum(["open", "closed"]), "status_notes": "string", "+service_name": "string", "+service_code": "string", "description": "string", "agency_responsible": "string", "service_notice": "string", "requested_datetime": valideer.AdaptBy(parser.parse), "updated_datetime": valideer.AdaptBy(parser.parse), "expected_datetime": valideer.AdaptBy(parser.parse), "address": "string", #TODO: Make Position XOR address "address_id": "string", "zipcode": "string", "+lat": valideer.AdaptTo(float, "number"), "+long": valideer.AdaptTo(float, "number"), "media_url": "string", }] #TODO: make URL regex } } def __init__(self, host, apikey, api=None): if (api is None): api = CitySDKParticipation(host, apikey) basic311.TestSuite.__init__(self, host, api) def testLocale(self, locale): print("Testing GET services (%s)" % locale), repl = self.api.getServices(locale) #TODO: Exact specs. for locale codes? expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") #Default and custom should differ replFields = self.xmlToDict(repl.content) if len(replFields["services"]) > 0: serviceLocale = replFields["services"]["service"][0] replFieldsEN = self.xmlToDict(self.cache_services.content) serviceEN = replFieldsEN["services"]["service"][0] if not (serviceLocale["service_name"] == serviceEN["service_name"]): print("ok") else: print("is ignored (same as defaults)") else: print "empty" def testRequestsExt(self): print("Testing GET extended requests"), # repl=self.cache_requests repl = self.api.getRequests() expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") replFields = self.xmlToDict(repl.content) print("- limits"), l = len(replFields["service_requests"]) if (l > 200): print("to much (%d)" % l) #90 days or first 200 else: print("ok") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_REQUESTSEXT) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) def testGetRequestsExtFiltered(self): print("Testing GET requests with extended filters") print("-geospatial"), repl = self.api.getRequests(lat=12.13955, long=54.09138, radius=2000) #TODO calculate that point self.testEmptyRequestSet(repl) print("-update_after") past = datetime.datetime.utcnow() - datetime.timedelta(7) repl = self.api.getRequests(updated_after=past.isoformat()) self.testEmptyRequestSet(repl) print("-update_after") repl = self.api.getRequests(updated_before=past.isoformat()) self.testEmptyRequestSet(repl)