def __init__( self, path, description_filename, installed_linked, schema_filename = None, latest_suitable_version = None, inherit_shrinkwrap = None ): # version, , represent versions and specifications, internal from yotta.lib import version # vcs, , represent version controlled directories, internal from yotta.lib import vcs # resolve links at creation time, to minimise path lengths: self.unresolved_path = path self.path = fsutils.realpath(path) self.installed_linked = installed_linked self.vcs = None self.error = None self.latest_suitable_version = latest_suitable_version self.version = None self.description_filename = description_filename self.ignore_list_fname = Ignore_List_Fname self.ignore_patterns = copy.copy(Default_Publish_Ignore) self.origin_info = None description_file = os.path.join(path, description_filename) if os.path.isfile(description_file): try: self.description = ordered_json.load(description_file) if self.description: if not 'name' in self.description: raise Exception('missing "name"') if 'version' in self.description: self.version = version.Version(self.description['version']) else: raise Exception('missing "version"') except Exception as e: self.description = OrderedDict() self.error = "Description invalid %s: %s" % (description_file, e); logger.debug(self.error) raise InvalidDescription(self.error) else: self.error = "No %s file." % description_filename self.description = OrderedDict() try: with open(os.path.join(path, self.ignore_list_fname), 'r') as ignorefile: self.ignore_patterns += self._parseIgnoreFile(ignorefile) except IOError as e: if e.errno != errno.ENOENT: raise # warn about invalid yotta versions before schema errors (as new yotta # might introduce new schema) yotta_version_spec = None if self.description and self.description.get('yotta', None): try: yotta_version_spec = version.Spec(self.description['yotta']) except ValueError as e: logger.warning( "could not parse yotta version spec '%s' from %s: it "+ "might require a newer version of yotta", self.description['yotta'], self.description['name'] ) if yotta_version_spec is not None: import yotta yotta_version = version.Version(yotta.__version__) if not yotta_version_spec.match(yotta_version): self.error = "requires yotta version %s (current version is %s). see http://docs.yottabuild.org for update instructions" % ( str(yotta_version_spec), str(yotta_version) ) if self.description and schema_filename and not self.path in self.schema_errors_displayed: self.schema_errors_displayed.add(self.path) have_errors = False with open(schema_filename, 'r') as schema_file: schema = json.load(schema_file) validator = jsonschema.Draft4Validator(schema) for error in validator.iter_errors(self.description): if not have_errors: logger.warning(u'%s has invalid %s:' % ( os.path.split(self.path.rstrip('/'))[1], description_filename )) have_errors = True logger.warning(u" %s value %s" % (u'.'.join([str(x) for x in error.path]), error.message)) # for now schema validation errors aren't fatal... will be soon # though! #if have_errors: # raise InvalidDescription('Invalid %s' % description_filename) self.inherited_shrinkwrap = None self.shrinkwrap = None # we can only apply shrinkwraps to instances with valid descriptions: # instances do not become valid after being invalid so this is safe # (but it means you cannot trust the shrinkwrap of an invalid # component) # (note that it is unsafe to use the __bool__ operator on self here as # we are not fully constructed) if self.description: self.inherited_shrinkwrap = inherit_shrinkwrap self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname), Shrinkwrap_Schema) if self.shrinkwrap: logger.warning('dependencies of %s are pegged by yotta-shrinkwrap.json', self.getName()) if self.inherited_shrinkwrap: logger.warning('shrinkwrap in %s overrides inherited shrinkwrap', self.getName()) #logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap) self.vcs = vcs.getVCS(path)
'enum': ['0.2.0'], 'description': 'The MicroDrop step version' }, 'additionalProperties': False } } }, } PROTOCOL_SCHEMA = copy.deepcopy(MESSAGE_SCHEMA) PROTOCOL_SCHEMA['allOf'] = [{'$ref': '#/definitions/protocol'}] STEP_SCHEMA = copy.deepcopy(MESSAGE_SCHEMA) STEP_SCHEMA['allOf'] = [{'$ref': '#/definitions/step'}] VALIDATORS = { 'protocol': jsonschema.Draft4Validator(PROTOCOL_SCHEMA), 'step': jsonschema.Draft4Validator(STEP_SCHEMA) } class SerializationError(Exception): ''' Attributes ---------- message : str Error message. exceptions : list List of objects corresponding to serialization exceptions. Objects are in the following form:
def test_validate_create(self): body = self.cluster schema = self.controller.get_schema('create', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
def validategeojson(data_input, mode): """GeoJSON validation example >>> import StringIO >>> class FakeInput(object): ... json = open('point.geojson','w') ... json.write('''{"type":"Feature", "properties":{}, "geometry":{"type":"Point", "coordinates":[8.5781228542328, 22.87500500679]}, "crs":{"type":"name", "properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"}}}''') # noqa ... json.close() ... file = 'point.geojson' >>> class fake_data_format(object): ... mimetype = 'application/geojson' >>> fake_input = FakeInput() >>> fake_input.data_format = fake_data_format() >>> validategeojson(fake_input, MODE.SIMPLE) True """ LOGGER.info('validating GeoJSON; Mode: %s', mode) passed = False if mode >= MODE.NONE: passed = True if mode >= MODE.SIMPLE: name = data_input.file (mtype, encoding) = mimetypes.guess_type(name, strict=False) passed = data_input.data_format.mime_type in { mtype, FORMATS.GEOJSON.mime_type } if mode >= MODE.STRICT: from pywps.dependencies import ogr data_source = ogr.Open(data_input.file) if data_source: passed = (data_source.GetDriver().GetName() == "GeoJSON") else: passed = False if mode >= MODE.VERYSTRICT: import jsonschema import json # this code comes from # https://github.com/om-henners/GeoJSON_Validation/blob/master/geojsonvalidation/geojson_validation.py schema_home = os.path.join(_get_schemas_home(), "geojson") base_schema = os.path.join(schema_home, "geojson.json") with open(base_schema) as fh: geojson_base = json.load(fh) with open(os.path.join(schema_home, "crs.json")) as fh: crs_json = json.load(fh) with open(os.path.join(schema_home, "bbox.json")) as fh: bbox_json = json.load(fh) with open(os.path.join(schema_home, "geometry.json")) as fh: geometry_json = json.load(fh) cached_json = { "http://json-schema.org/geojson/crs.json": crs_json, "http://json-schema.org/geojson/bbox.json": bbox_json, "http://json-schema.org/geojson/geometry.json": geometry_json } resolver = jsonschema.RefResolver( "http://json-schema.org/geojson/geojson.json", geojson_base, store=cached_json) validator = jsonschema.Draft4Validator(geojson_base, resolver=resolver) try: validator.validate(json.loads(data_input.stream.read())) passed = True except jsonschema.ValidationError: passed = False return passed
def validate_cube(self, cube): validator = jsonschema.Draft4Validator(self.cube_schema) name = cube.get("name") return self._collect_errors("cube", name, validator, cube)
def run(*, src: str, schema: str) -> None: src = loading.loadfile(src) schema = loading.loadfile(schema) validator = jsonschema.Draft4Validator(schema) print("@", validator.validate(src))
def clean(self): try: jsonschema.Draft4Validator(self._schema).validate(self.input) except SchemaError as e: raise ValidationError({'input': e.message})
def test_validate_resize_instance_string(self): body = {"resize": {"flavorRef": 'foo'}} schema = self.controller.get_schema('action', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
def w(*args, **kwargs): schema = f(*args, **kwargs) x = jsonschema.Draft4Validator(schema) return schema
def test_validate_resize_volume_string(self): body = {"resize": {"volume": {"size": '-44.0'}}} schema = self.controller.get_schema('action', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
def test_validate_resize_instance(self): body = {"resize": {"flavorRef": "https://endpoint/v1.0/123/flavors/2"}} schema = self.controller.get_schema('action', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
def test_validate_invalid_action(self): # TODO(juice) perhaps we should validate the schema not recognized body = {"restarted": {}} schema = self.controller.get_schema('action', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
def test_validate_restart(self): body = {"restart": {}} schema = self.controller.get_schema('action', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
PROPERTIES = "properties" # This validator will set default values in properties. # This does not return a complete set of errors; use only for setting defaults. # Pass this object a schema to get a validator for that schema. DEFAULT_SETTER = schema_validation_utils.ExtendWithDefault( jsonschema.Draft4Validator) # This is a regular validator, use after using the DEFAULT_SETTER # Pass this object a schema to get a validator for that schema. VALIDATOR = jsonschema.Draft4Validator # This is a validator using the default Draft4 metaschema, # use it to validate user schemas. SCHEMA_VALIDATOR = jsonschema.Draft4Validator( jsonschema.Draft4Validator.META_SCHEMA) # JsonSchema to be used to validate the user's "imports:" section IMPORT_SCHEMA = """ properties: imports: type: array items: type: object required: - path properties: path: type: string name: type: string
def load_project_data(source_directory, check_all_links=False, skip_links=False, series_to_load=None, governed_deliverables=[], strict=False, projects_to_check=[]): "Return a dict with project data grouped by series." logger = logging.getLogger() series_to_load = series_to_load or [] project_data = {} fail = False service_types = os_service_types.ServiceTypes(session=requests.Session(), only_remote=True) # Set up a schema validator so we can quickly check that the input # data conforms. project_schema_filename = os.path.join( source_directory, 'project-data', 'schema.yaml', ) with open(project_schema_filename, 'r') as f: project_schema = yaml.safe_load(f.read()) validator = jsonschema.Draft4Validator(project_schema) # Load the data files, using the file basename as the release # series name. for filename in glob.glob( os.path.join(source_directory, 'project-data', '*.yaml')): if filename.endswith('schema.yaml'): continue series, _ = os.path.splitext(os.path.basename(filename)) if series_to_load and series not in series_to_load: continue logger.info('loading %s project data from %s', series, filename) with open(filename, 'r') as f: raw_data = yaml.safe_load(f.read()) for error in validator.iter_errors(raw_data): logger.error(str(error)) fail = True links_to_check = [] data = [] for project in raw_data: deliverable_name = project.get('deliverable-name', project['name']) # Set the defaults for the flags so that the templates can # assume the flags with true defaults are defined. for url_info in _URLS: if url_info.flag_name not in project: project[url_info.flag_name] = url_info.default if (series == 'latest' and deliverable_name not in governed_deliverables): msg = ('{} is no longer part of an official project, ' '{} in {}').format(deliverable_name, 'error' if strict else 'ignoring', filename) logger.warning(msg) if strict: logger.info('Known deliverables: %s', sorted(governed_deliverables)) raise RuntimeError(msg) continue logger.info('including %s', deliverable_name) data.append(project) # If the project has a service-type set, ensure it matches # the value in the service-type-authority data.base. st = project.get('service_type') if st is not None: st_data = service_types.get_service_data_for_project( project['name']) if not st_data: # It's possible this is a project listed by its # service-type st_data = service_types.get_service_data(st) if not st_data: logger.error( 'did not find %s in Service Types Authority', project['name'], ) fail = True elif st != st_data['service_type']: logger.error( 'expected service_type %r for %s but got %r', st_data['service_type'], project['name'], st, ) fail = True # client projects must have a description project_type = project.get('type') if (project_type in ['cloud-client', 'service-client'] and not project.get('description')): logger.error( 'client project %s has no description', project['name'], ) fail = True # If the project claims to have a separately published guide # of some sort, look for it before allowing the flag to stand. check_links_this_project = (deliverable_name in projects_to_check or not projects_to_check) if check_links_this_project and not skip_links: for url_info in _URLS: flag_val = project.get(url_info.flag_name, url_info.default) if ((not flag_val) and url_info.types and project_type not in url_info.types): # This type of project isn't expected to have # this type of link, so if we are not # explicitly told to check for it don't. continue try: url = url_info.template.format(series=series, **project) except KeyError: # The project data does not include a field needed # to build the URL (typically the # service_type). Ignore this URL, unless the flag # is set. if flag_val: raise continue # Only try to fetch the URL if we're going to do # something with the result. if flag_val or check_all_links: logger.info('%s:%s looking for %s', series, project['name'], url) links_to_check.append((url, project['name'], url_info.flag_name, flag_val)) if links_to_check: logger.info('checking %s links from %s...', len(links_to_check), filename) pool = multiprocessing.pool.ThreadPool() results = pool.map(_check_url, links_to_check) for url, project_name, flag, flag_val, exists, status in results: if flag_val and not exists: logger.error( '%s set for %s but %s does not exist (%s)', flag, project_name, url, status, ) fail = True elif (not flag_val) and check_all_links and exists: msg = '{} not set for {} but {} does exist'.format( flag, project_name, url) logger.warning(msg) if strict: raise RuntimeError(msg) if fail: raise ValueError('invalid input in %s' % filename) project_data[series] = data return project_data
def test_validate_upgrade(self): body = self.upgrade schema = self.controller.get_schema('action', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
import json import pkgutil import re try: # Python 2 from collections import Mapping, Sequence except ImportError: # Python 3 from collections.abc import Mapping, Sequence import jsonschema from funcsigs import signature from .exceptions import InvalidParams, InvalidRequest, MethodNotFound _JSON_VALIDATOR = jsonschema.Draft4Validator( json.loads(pkgutil.get_data(__name__, 'request-schema.json').decode())) def convert_camel_case(name): """Convert a camelCase string to under_score""" string = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', string).lower() def convert_camel_case_keys(original_dict): """Converts all keys of a dict from camelCase to under_score, recursively""" new_dict = dict() for key, val in original_dict.items(): if isinstance(val, dict): # Recurse new_dict[convert_camel_case(key)] = convert_camel_case_keys(val)
def parse_config(configfile, refpath=None, testpath=None): print('\n=====CONFIGURATION=====') # check inputs if configfile and not os.path.isfile(configfile): raise IOError("Configuration file does not exist") if refpath and not os.path.isdir(refpath): raise IOError('"refpath" not a valid folder <{}>'.format(refpath)) if testpath and not os.path.isdir(testpath): raise IOError('"testpath" not a valid folder <{}>'.format(testpath)) # create schema validator object (& check schema itself) schema = json.loads( pkg_resources.resource_string(resource_package, 'config_schema.json').decode('utf-8')) validator = jsonschema.Draft4Validator(schema) validator.check_schema(schema) # load user configuration print("\nReading configuration from <{}>".format(configfile)) # JSON parsing if configfile.endswith(('.json', '.JSON')): # open & read JSON file with open(configfile, 'r') as fid: config = json.load(fid) # CONFIG parsing elif configfile.endswith(('.config', '.CONFIG')): # setup config parser parser = configparser.ConfigParser() parser.optionxform = str # maintain case-sensitive items # read entire configuration file into dict if len(parser.read(configfile)) == 0: raise IOError("Unable to read selected .config file") config = {s: dict(parser.items(s)) for s in parser.sections()} # special section/item parsing s = 'INPUT.REF' i = 'CLSMatchValue' config[s][i] = ast.literal_eval(config[s][i]) s = 'INPUT.TEST' i = 'CLSMatchValue' if i in config[s]: # Optional Field config[s][i] = ast.literal_eval(config[s][i]) else: config[s][i] = config['INPUT.REF'][i] # bool(config[s][i]) does not interpret 'true'/'false' strings s = 'OPTIONS' i = 'QuantizeHeight' config[s][i] = parser.getboolean(s, i) s = 'PLOTS' i = 'ShowPlots' config[s][i] = parser.getboolean(s, i) s = 'PLOTS' i = 'SavePlots' config[s][i] = parser.getboolean(s, i) s = 'MATERIALS.REF' i = 'MaterialNames' config[s][i] = config[s][i].split(',') s = 'MATERIALS.REF' i = 'MaterialIndicesToIgnore' config[s][i] = [int(v) for v in config[s][i].split(',')] # unrecognized config file type else: raise IOError('Unrecognized configuration file') # locate files for each "xxxFilename" configuration parameter # this makes use of "refpath" and "testpath" arguments for relative filenames # we do this before validation to ensure required files are located for item in [('INPUT.REF', refpath), ('INPUT.TEST', testpath)]: sec = item[0] path = item[1] print('\nPROCESSING "{}" FILES'.format(sec)) config[sec] = findfiles(config[sec], path) # validate final configuration against schema try: validator.validate(config) print('\nCONFIGURATION VALIDATED') except jsonschema.exceptions.ValidationError: print('\n*****INVALID CONFIGURATION FILE*****\n') for error in sorted(validator.iter_errors(config), key=str): print('ERROR: {}\n'.format(error)) raise jsonschema.exceptions.ValidationError('validation error') # for easier explotation, ensure some configuration options are tuple/list opts = (('INPUT.TEST', 'CLSMatchValue'), ('INPUT.REF', 'CLSMatchValue'), ('MATERIALS.REF', 'MaterialIndicesToIgnore')) for opt in opts: s = opt[0] i = opt[1] try: _ = (v for v in config[s][i]) except: config[s][i] = [config[s][i]] # print final configuration print('\nFINAL CONFIGURATION') print(json.dumps(config, indent=2)) # cleanup return config
import enhancements from http import HttpConnection from opsgenie import OpsGenieAlerter import ruletypes from util import dt_to_ts from util import dt_to_ts_with_format from util import dt_to_unix from util import dt_to_unixms from util import EAException from util import ts_to_dt from util import ts_to_dt_with_format from util import unix_to_dt from util import unixms_to_dt # schema for rule yaml rule_schema = jsonschema.Draft4Validator( yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) # Required global (config.yaml) and local (rule.yaml) configuration options required_globals = frozenset([ 'run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time' ]) required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables env_settings = { 'ES_USE_SSL': 'use_ssl', 'ES_PASSWORD': '******', 'ES_USERNAME': '******', 'ES_HOST': 'es_host', 'ES_PORT': 'es_port'
def check_scoring(json_conf): """ :param json_conf: configuration dictionary to check. :type json_conf: dict Function to check the "scoring" section of the configuration. Each scoring function will be checked for: - validity of the expression (it can be interpreted by Mikado) - validity of the parameter (it is a valid Metric) :return: json_conf :rtype dict """ with io.TextIOWrapper(resource_stream(__name__, "scoring_blueprint.json")) as schema: scoring_schema = json.load(schema) parameters_found = set() parameters_not_found = [] double_parameters = [] invalid_filter = set() available_metrics = Transcript.get_available_metrics() if "scoring" not in json_conf or len(json_conf["scoring"].keys()) == 0: raise InvalidJson("No parameters specified for scoring!") validator = extend_with_default(jsonschema.Draft4Validator) for parameter in json_conf["scoring"]: if parameter not in available_metrics: parameters_not_found.append(parameter) if parameter in parameters_found: double_parameters.append(parameter) if not jsonschema.Draft4Validator(scoring_schema).is_valid( json_conf["scoring"][parameter]): errors = [ str(_) for _ in list( jsonschema.Draft4Validator(scoring_schema).iter_errors( json_conf["scoring"][parameter])) ] raise InvalidJson("Invalid scoring for {}:\n{}".format( parameter, "\n".join(errors))) try: validator(scoring_schema).validate(json_conf["scoring"][parameter]) except Exception as err: raise ValueError(parameter, err) if json_conf["scoring"][parameter]["rescaling"] == "target": if "value" not in json_conf["scoring"][parameter]: message = """Target rescaling requested for {0} but no target value specified. Please specify it with the \"value\" keyword.\n{1}""" message = message.format(parameter, json_conf["scoring"][parameter]) raise UnrecognizedRescaler(message) if len(parameters_not_found) > 0 or len(double_parameters) > 0 or len( invalid_filter) > 0: err_message = '' if len(parameters_not_found) > 0: err_message = """The following parameters, present in the JSON file, are not available!\n\t{0}\n""".format( "\n\t".join(parameters_not_found)) if len(double_parameters) > 0: err_message += """The following parameters have been specified more than once, please correct:\n\t{0}""".format("\n\t".join( list(double_parameters))) if len(invalid_filter) > 0: err_message += """The following parameters have an invalid filter, please correct: \t{0}""".format("\n\t".join(list(invalid_filter))) raise InvalidJson(err_message) return json_conf
def __init__(self): self._schema = jsonschema.Draft4Validator(CONFIG_SCHEMA)
def check_requirements(json_conf, require_schema, index): """ Function to check the "requirements" section of the configuration. Each filtering function will be checked for: - validity of the expression (it can be interpreted by Mikado) - validity of the parameter (it is a valid Metric) :param json_conf: configuration dictionary to check. :type json_conf: dict :param require_schema: the requirements section of the JSON schema. :type require_schema: dict :return: json_conf :rtype dict """ # Check that the parameters are valid parameters_not_found = [] available_metrics = Transcript.get_available_metrics() if "parameters" not in json_conf[index]: raise InvalidJson( "The {} field must have a \"parameters\" subfield!".format(index)) for key in json_conf[index]["parameters"]: key_name = key.split(".")[0] if key_name not in available_metrics: parameters_not_found.append(key_name) continue if not jsonschema.Draft4Validator( require_schema["definitions"]["parameter"]).is_valid( json_conf[index]["parameters"][key]): errors = list( jsonschema.Draft4Validator(require_schema).iter_errors( json_conf[index]["parameters"][key])) raise InvalidJson("Invalid parameter for {0} in {1}: \n{2}".format( key, index, errors)) json_conf[index]["parameters"][key]["name"] = key_name if len(parameters_not_found) > 0: raise InvalidJson( "The following parameters, selected for filtering, are invalid:\n\t{0}" .format("\n\t".join(parameters_not_found))) # Create automatically a filtering expression if "expression" not in json_conf[index]: json_conf[index]["expression"] = " and ".join( list(json_conf[index]["parameters"].keys())) keys = json_conf[index]["parameters"].keys() newexpr = json_conf[index]["expression"][:] else: if not jsonschema.Draft4Validator( require_schema["definitions"]["expression"]).is_valid( json_conf[index]["expression"]): raise InvalidJson("Invalid expression field") expr = " ".join(json_conf[index]["expression"]) newexpr = expr[:] keys = list(key for key in re.findall("([^ ()]+)", expr) if key not in ("and", "or", "not", "xor")) diff_params = set.difference( set(keys), set(json_conf[index]["parameters"].keys())) if len(diff_params) > 0: raise InvalidJson( "Expression and required parameters mismatch:\n\t{0}".format( "\n\t".join(list(diff_params)))) for key in keys: # Create the final expression newexpr = re.sub(key, "evaluated[\"{0}\"]".format(key), newexpr) # Test the expression try: compile(newexpr, "<json>", "eval") except SyntaxError: raise InvalidJson("Invalid expression for {}:\n{}".format( index, newexpr)) json_conf[index]["expression"] = newexpr return json_conf
def test_validate_create_complete(self): body = {"backup": {"instance": self.uuid, "name": "testback-backup"}} schema = self.controller.get_schema('create', body) validator = jsonschema.Draft4Validator(schema) self.assertTrue(validator.is_valid(body))
import jsonschema import pkg_resources try: import simplejson as json except ImportError: import json _FORM_SCHEMA_JSON = json.loads( pkg_resources.resource_string( "slivka", "data/config/FormDescriptionSchema.json" ).decode() ) jsonschema.Draft4Validator.check_schema(_FORM_SCHEMA_JSON) FORM_VALIDATOR = jsonschema.Draft4Validator(_FORM_SCHEMA_JSON) _COMMAND_SCHEMA_JSON = json.loads( pkg_resources.resource_string( "slivka", "data/config/ConfDescriptionSchema.json" ).decode() ) jsonschema.Draft4Validator.check_schema(_COMMAND_SCHEMA_JSON) COMMAND_VALIDATOR = jsonschema.Draft4Validator(_COMMAND_SCHEMA_JSON) class Singleton(type): __instances = {} def __call__(cls, *args, **kwargs):
class Server(with_metaclass(ABCMeta, object)): """Protocol-agnostic class representing the remote server. Subclasses should inherit and override ``_send_message``. :param endpoint: The server address. """ # Request and response logs __request_log = logging.getLogger(__name__ + '.request') __response_log = logging.getLogger(__name__ + '.response') #: Validate the response message __validator = jsonschema.Draft4Validator( json.loads( pkgutil.get_data(__name__, 'response-schema.json').decode('utf-8'))) def __init__(self, endpoint): #: Holds the server address self.endpoint = endpoint def _log_request(self, request, extra=None): """Log the JSON-RPC request before sending. Should be called by subclasses in :meth:`_send_message`, before sending. :param request: The JSON-RPC request string. :param extra: A dict of extra fields that may be logged. """ if extra is None: extra = {} # Add endpoint to list of info to include in log message extra.update({'endpoint': self.endpoint}) _log(self.__request_log, 'info', request, fmt='--> %(message)s', extra=extra) def _log_response(self, response, extra=None): """Log the JSON-RPC response after sending. Should be called by subclasses in :meth:`_send_message`, after receiving the response. :param response: The JSON-RPC response string. :param extra: A dict of extra fields that may be logged. """ if extra is None: extra = {} # Add the endpoint to the log entry extra.update({'endpoint': self.endpoint}) # Clean up the response for logging response = response.replace("\n", '').replace(' ', ' ') \ .replace('{ ', '{') _log(self.__response_log, 'info', response, fmt='<-- %(message)s', extra=extra) def _process_response(self, response): """Processes the response and returns the 'result' portion if present. :param response: The JSON-RPC response string to process. :return: The response string, or None """ if response: if isinstance(response, basestring): # Attempt to parse the response try: response = json.loads(response) except ValueError: raise exceptions.ParseResponseError() # Validate the response against the Response schema (raises # jsonschema.ValidationError if invalid) if config.validate: self.__validator.validate(response) if isinstance(response, list): # For now, just return the whole response return response else: # If the response was "error", raise to ensure it's handled if 'error' in response: raise exceptions.ReceivedErrorResponse( response['error'].get('code'), response['error'].get('message'), response['error'].get('data')) # else return response.get('result') # No response was given return None @abstractmethod def _send_message(self, request, **kwargs): """Used internally - send the request to the server. Override this method in the protocol-specific subclasses. Be sure to log both the request and response, and return the response. :param request: A JSON-RPC request, in dict format. :returns: The response (a string for requests, None for notifications). """ def send(self, request, **kwargs): """Send a request, passing the whole JSON-RPC `request object <http://www.jsonrpc.org/specification#request_object>`_. >>> server.send({'jsonrpc': '2.0', 'method': 'ping', 'id': 1}) --> {"jsonrpc": "2.0", "method": "ping", "id": 1} <-- {"jsonrpc": "2.0", "result": "pong", "id": 1} 'pong' :param request: The JSON-RPC request. :type request: string or a JSON serializable object :param kwargs: For HTTPServer, these are passed on to `requests.Session.send() <http://docs.python-requests.org/en/master/api/#requests.Session.send>`_. :returns: The payload (i.e. the ``result`` part of the response, or ``None`` in the case of a Notification). :rtype: A `JSON-decoded object <https://docs.python.org/library/json.html#json-to-py-table>`_. :raises ParseResponseError: The response was not valid JSON. :raises ValidationError: The response was valid JSON, but not valid JSON-RPC. :raises ReceivedErrorResponse: The server responded with a JSON-RPC `error object <http://www.jsonrpc.org/specification#error_object>`_. """ # Convert request to a string if not isinstance(request, basestring): request = json.dumps(request) # Call internal method to transport the message response = self._send_message(request, **kwargs) return self._process_response(response) # Alternate ways to send a request ----------- def notify(self, method_name, *args, **kwargs): """Send a JSON-RPC request, without expecting a response. :param method_name: The remote procedure's method name. :param args: Positional arguments passed to the remote procedure. :param kwargs: Keyword arguments passed to the remote procedure. :return: The payload (i.e. the ``result`` part of the response). """ return self.send(Notification(method_name, *args, **kwargs)) def request(self, method_name, *args, **kwargs): """Send a request, by passing the method and arguments. This is the main public method. >>> server.request('cat', name='Mittens') --> {"jsonrpc": "2.0", "method": "cat", "params": {"name": "Mittens"}, "id": 1} <-- {"jsonrpc": "2.0", "result": "meow", "id": 1} 'meow' :param method_name: The remote procedure's method name. :param args: Positional arguments passed to the remote procedure. :param kwargs: Keyword arguments passed to the remote procedure. :return: The payload (i.e. the ``result`` part of the response). """ return self.send(Request(method_name, *args, **kwargs)) def __getattr__(self, name): """This gives us an alternate way to make a request:: >>> server.cube(3) 27 That's the same as saying ``server.request('cube', 3)``. Technique is explained here: http://code.activestate.com/recipes/307618/ """ def attr_handler(*args, **kwargs): """Call self.request from here""" return self.request(name, *args, **kwargs) return attr_handler
def match(schema: dict, response: Any) -> List[dict]: validator = jsonschema.Draft4Validator(schema) errors = list(validator.iter_errors(response)) return [dict(value=list(e.path), message=e.message) for e in errors]
def _validate_jsonschema(instance, schema): validator = jsonschema.Draft4Validator(schema) errors = list(validator.iter_errors(instance)) if len(errors) != 0: sys.exit("\tERROR\n\nValidation error: {}".format(errors))
ERRORS_SCHEMA = os.path.join(cur_dir, '.schemacache', 'errors', 'payload.json') TRANSACTIONS_SCHEMA = os.path.join(cur_dir, '.schemacache', 'transactions', 'payload.json') assert (os.path.exists(ERRORS_SCHEMA) and os.path.exists(TRANSACTIONS_SCHEMA)), \ 'JSON Schema files not found. Run "make update-json-schema to download' with codecs.open(ERRORS_SCHEMA, encoding='utf8') as errors_json, \ codecs.open(TRANSACTIONS_SCHEMA, encoding='utf8') as transactions_json: VALIDATORS = { '/v1/errors': jsonschema.Draft4Validator( json.load(errors_json), resolver=jsonschema.RefResolver( base_uri='file:' + pathname2url(ERRORS_SCHEMA), referrer='file:' + pathname2url(ERRORS_SCHEMA), )), '/v1/transactions': jsonschema.Draft4Validator( json.load(transactions_json), resolver=jsonschema.RefResolver( base_uri='file:' + pathname2url(TRANSACTIONS_SCHEMA), referrer='file:' + pathname2url(TRANSACTIONS_SCHEMA), )) } class ValidatingWSGIApp(ContentServer): def __init__(self, **kwargs): super(ValidatingWSGIApp, self).__init__(**kwargs)
def validate_armada_document(document): """Validates a document ingested by Armada by subjecting it to JSON schema validation. :param dict dictionary: The document to validate. :returns: A tuple of (bool, list[dict]) where the first value indicates whether the validation succeeded or failed and the second value is the validation details with a minimum keyset of (message(str), error(bool)) :rtype: tuple. :raises TypeError: If ``document`` is not of type ``dict``. """ if not isinstance(document, dict): raise TypeError('The provided input "%s" must be a dictionary.' % document) schema = document.get('schema', '<missing>') document_name = document.get('metadata', {}).get('name', None) details = [] LOG.debug('Validating document [%s] %s', schema, document_name) if schema in SCHEMAS: try: validator = jsonschema.Draft4Validator(SCHEMAS[schema]) for error in validator.iter_errors(document.get('data')): error_message = "Invalid document [%s] %s: %s." % \ (schema, document_name, error.message) vmsg = ValidationMessage(message=error_message, error=True, name='ARM100', level='Error', schema=schema, doc_name=document_name) LOG.info('ValidationMessage: %s', vmsg.get_output_json()) details.append(vmsg.get_output()) except jsonschema.SchemaError as e: error_message = ('The built-in Armada JSON schema %s is invalid. ' 'Details: %s.' % (e.schema, e.message)) vmsg = ValidationMessage(message=error_message, error=True, name='ARM000', level='Error', diagnostic='Armada is misconfigured.') LOG.error('ValidationMessage: %s', vmsg.get_output_json()) details.append(vmsg.get_output()) else: vmsg = ValidationMessage(message='Unsupported document type.', error=False, name='ARM002', level='Warning', schema=schema, doc_name=document_name, diagnostic='Please ensure document is one of ' 'the following schema types: %s' % list(SCHEMAS.keys())) LOG.info('Unsupported document type, ignoring %s.', schema) LOG.debug('ValidationMessage: %s', vmsg.get_output_json()) # Validation API doesn't care about this type of message, don't send if len([x for x in details if x.get('error', False)]) > 0: return False, details return True, details
def validate_training_request(request): """Validate a JSON-formatted training request.""" return jsonschema.Draft4Validator(schema).is_valid(request)