def __init__(
self,
path,
description_filename,
installed_linked,
schema_filename = None,
latest_suitable_version = None,
inherit_shrinkwrap = None
):
# version, , represent versions and specifications, internal
from yotta.lib import version
# vcs, , represent version controlled directories, internal
from yotta.lib import vcs
# resolve links at creation time, to minimise path lengths:
self.unresolved_path = path
self.path = fsutils.realpath(path)
self.installed_linked = installed_linked
self.vcs = None
self.error = None
self.latest_suitable_version = latest_suitable_version
self.version = None
self.description_filename = description_filename
self.ignore_list_fname = Ignore_List_Fname
self.ignore_patterns = copy.copy(Default_Publish_Ignore)
self.origin_info = None
description_file = os.path.join(path, description_filename)
if os.path.isfile(description_file):
try:
self.description = ordered_json.load(description_file)
if self.description:
if not 'name' in self.description:
raise Exception('missing "name"')
if 'version' in self.description:
self.version = version.Version(self.description['version'])
else:
raise Exception('missing "version"')
except Exception as e:
self.description = OrderedDict()
self.error = "Description invalid %s: %s" % (description_file, e);
logger.debug(self.error)
raise InvalidDescription(self.error)
else:
self.error = "No %s file." % description_filename
self.description = OrderedDict()
try:
with open(os.path.join(path, self.ignore_list_fname), 'r') as ignorefile:
self.ignore_patterns += self._parseIgnoreFile(ignorefile)
except IOError as e:
if e.errno != errno.ENOENT:
raise
# warn about invalid yotta versions before schema errors (as new yotta
# might introduce new schema)
yotta_version_spec = None
if self.description and self.description.get('yotta', None):
try:
yotta_version_spec = version.Spec(self.description['yotta'])
except ValueError as e:
logger.warning(
"could not parse yotta version spec '%s' from %s: it "+
"might require a newer version of yotta",
self.description['yotta'],
self.description['name']
)
if yotta_version_spec is not None:
import yotta
yotta_version = version.Version(yotta.__version__)
if not yotta_version_spec.match(yotta_version):
self.error = "requires yotta version %s (current version is %s). see http://docs.yottabuild.org for update instructions" % (
str(yotta_version_spec),
str(yotta_version)
)
if self.description and schema_filename and not self.path in self.schema_errors_displayed:
self.schema_errors_displayed.add(self.path)
have_errors = False
with open(schema_filename, 'r') as schema_file:
schema = json.load(schema_file)
validator = jsonschema.Draft4Validator(schema)
for error in validator.iter_errors(self.description):
if not have_errors:
logger.warning(u'%s has invalid %s:' % (
os.path.split(self.path.rstrip('/'))[1],
description_filename
))
have_errors = True
logger.warning(u" %s value %s" % (u'.'.join([str(x) for x in error.path]), error.message))
# for now schema validation errors aren't fatal... will be soon
# though!
#if have_errors:
# raise InvalidDescription('Invalid %s' % description_filename)
self.inherited_shrinkwrap = None
self.shrinkwrap = None
# we can only apply shrinkwraps to instances with valid descriptions:
# instances do not become valid after being invalid so this is safe
# (but it means you cannot trust the shrinkwrap of an invalid
# component)
# (note that it is unsafe to use the __bool__ operator on self here as
# we are not fully constructed)
if self.description:
self.inherited_shrinkwrap = inherit_shrinkwrap
self.shrinkwrap = tryReadJSON(os.path.join(path, Shrinkwrap_Fname), Shrinkwrap_Schema)
if self.shrinkwrap:
logger.warning('dependencies of %s are pegged by yotta-shrinkwrap.json', self.getName())
if self.inherited_shrinkwrap:
logger.warning('shrinkwrap in %s overrides inherited shrinkwrap', self.getName())
#logger.info('%s created with inherited_shrinkwrap %s', self.getName(), self.inherited_shrinkwrap)
self.vcs = vcs.getVCS(path)
'enum': ['0.2.0'],
'description': 'The MicroDrop step version'
},
'additionalProperties': False
}
}
},
}
PROTOCOL_SCHEMA = copy.deepcopy(MESSAGE_SCHEMA)
PROTOCOL_SCHEMA['allOf'] = [{'$ref': '#/definitions/protocol'}]
STEP_SCHEMA = copy.deepcopy(MESSAGE_SCHEMA)
STEP_SCHEMA['allOf'] = [{'$ref': '#/definitions/step'}]
VALIDATORS = {
'protocol': jsonschema.Draft4Validator(PROTOCOL_SCHEMA),
'step': jsonschema.Draft4Validator(STEP_SCHEMA)
}
class SerializationError(Exception):
'''
Attributes
----------
message : str
Error message.
exceptions : list
List of objects corresponding to serialization exceptions.
Objects are in the following form:
def test_validate_create(self):
body = self.cluster
schema = self.controller.get_schema('create', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
def validategeojson(data_input, mode):
"""GeoJSON validation example
>>> import StringIO
>>> class FakeInput(object):
... json = open('point.geojson','w')
... json.write('''{"type":"Feature", "properties":{}, "geometry":{"type":"Point", "coordinates":[8.5781228542328, 22.87500500679]}, "crs":{"type":"name", "properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"}}}''') # noqa
... json.close()
... file = 'point.geojson'
>>> class fake_data_format(object):
... mimetype = 'application/geojson'
>>> fake_input = FakeInput()
>>> fake_input.data_format = fake_data_format()
>>> validategeojson(fake_input, MODE.SIMPLE)
True
"""
LOGGER.info('validating GeoJSON; Mode: %s', mode)
passed = False
if mode >= MODE.NONE:
passed = True
if mode >= MODE.SIMPLE:
name = data_input.file
(mtype, encoding) = mimetypes.guess_type(name, strict=False)
passed = data_input.data_format.mime_type in {
mtype, FORMATS.GEOJSON.mime_type
}
if mode >= MODE.STRICT:
from pywps.dependencies import ogr
data_source = ogr.Open(data_input.file)
if data_source:
passed = (data_source.GetDriver().GetName() == "GeoJSON")
else:
passed = False
if mode >= MODE.VERYSTRICT:
import jsonschema
import json
# this code comes from
# https://github.com/om-henners/GeoJSON_Validation/blob/master/geojsonvalidation/geojson_validation.py
schema_home = os.path.join(_get_schemas_home(), "geojson")
base_schema = os.path.join(schema_home, "geojson.json")
with open(base_schema) as fh:
geojson_base = json.load(fh)
with open(os.path.join(schema_home, "crs.json")) as fh:
crs_json = json.load(fh)
with open(os.path.join(schema_home, "bbox.json")) as fh:
bbox_json = json.load(fh)
with open(os.path.join(schema_home, "geometry.json")) as fh:
geometry_json = json.load(fh)
cached_json = {
"http://json-schema.org/geojson/crs.json": crs_json,
"http://json-schema.org/geojson/bbox.json": bbox_json,
"http://json-schema.org/geojson/geometry.json": geometry_json
}
resolver = jsonschema.RefResolver(
"http://json-schema.org/geojson/geojson.json",
geojson_base,
store=cached_json)
validator = jsonschema.Draft4Validator(geojson_base, resolver=resolver)
try:
validator.validate(json.loads(data_input.stream.read()))
passed = True
except jsonschema.ValidationError:
passed = False
return passed
def validate_cube(self, cube):
validator = jsonschema.Draft4Validator(self.cube_schema)
name = cube.get("name")
return self._collect_errors("cube", name, validator, cube)
def run(*, src: str, schema: str) -> None:
src = loading.loadfile(src)
schema = loading.loadfile(schema)
validator = jsonschema.Draft4Validator(schema)
print("@", validator.validate(src))
def clean(self):
try:
jsonschema.Draft4Validator(self._schema).validate(self.input)
except SchemaError as e:
raise ValidationError({'input': e.message})
def test_validate_resize_instance_string(self):
body = {"resize": {"flavorRef": 'foo'}}
schema = self.controller.get_schema('action', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
def w(*args, **kwargs):
schema = f(*args, **kwargs)
x = jsonschema.Draft4Validator(schema)
return schema
def test_validate_resize_volume_string(self):
body = {"resize": {"volume": {"size": '-44.0'}}}
schema = self.controller.get_schema('action', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
def test_validate_resize_instance(self):
body = {"resize": {"flavorRef": "https://endpoint/v1.0/123/flavors/2"}}
schema = self.controller.get_schema('action', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
def test_validate_invalid_action(self):
# TODO(juice) perhaps we should validate the schema not recognized
body = {"restarted": {}}
schema = self.controller.get_schema('action', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
def test_validate_restart(self):
body = {"restart": {}}
schema = self.controller.get_schema('action', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
PROPERTIES = "properties"
# This validator will set default values in properties.
# This does not return a complete set of errors; use only for setting defaults.
# Pass this object a schema to get a validator for that schema.
DEFAULT_SETTER = schema_validation_utils.ExtendWithDefault(
jsonschema.Draft4Validator)
# This is a regular validator, use after using the DEFAULT_SETTER
# Pass this object a schema to get a validator for that schema.
VALIDATOR = jsonschema.Draft4Validator
# This is a validator using the default Draft4 metaschema,
# use it to validate user schemas.
SCHEMA_VALIDATOR = jsonschema.Draft4Validator(
jsonschema.Draft4Validator.META_SCHEMA)
# JsonSchema to be used to validate the user's "imports:" section
IMPORT_SCHEMA = """
properties:
imports:
type: array
items:
type: object
required:
- path
properties:
path:
type: string
name:
type: string
def load_project_data(source_directory,
check_all_links=False,
skip_links=False,
series_to_load=None,
governed_deliverables=[],
strict=False,
projects_to_check=[]):
"Return a dict with project data grouped by series."
logger = logging.getLogger()
series_to_load = series_to_load or []
project_data = {}
fail = False
service_types = os_service_types.ServiceTypes(session=requests.Session(),
only_remote=True)
# Set up a schema validator so we can quickly check that the input
# data conforms.
project_schema_filename = os.path.join(
source_directory,
'project-data',
'schema.yaml',
)
with open(project_schema_filename, 'r') as f:
project_schema = yaml.safe_load(f.read())
validator = jsonschema.Draft4Validator(project_schema)
# Load the data files, using the file basename as the release
# series name.
for filename in glob.glob(
os.path.join(source_directory, 'project-data', '*.yaml')):
if filename.endswith('schema.yaml'):
continue
series, _ = os.path.splitext(os.path.basename(filename))
if series_to_load and series not in series_to_load:
continue
logger.info('loading %s project data from %s', series, filename)
with open(filename, 'r') as f:
raw_data = yaml.safe_load(f.read())
for error in validator.iter_errors(raw_data):
logger.error(str(error))
fail = True
links_to_check = []
data = []
for project in raw_data:
deliverable_name = project.get('deliverable-name', project['name'])
# Set the defaults for the flags so that the templates can
# assume the flags with true defaults are defined.
for url_info in _URLS:
if url_info.flag_name not in project:
project[url_info.flag_name] = url_info.default
if (series == 'latest'
and deliverable_name not in governed_deliverables):
msg = ('{} is no longer part of an official project, '
'{} in {}').format(deliverable_name,
'error' if strict else 'ignoring',
filename)
logger.warning(msg)
if strict:
logger.info('Known deliverables: %s',
sorted(governed_deliverables))
raise RuntimeError(msg)
continue
logger.info('including %s', deliverable_name)
data.append(project)
# If the project has a service-type set, ensure it matches
# the value in the service-type-authority data.base.
st = project.get('service_type')
if st is not None:
st_data = service_types.get_service_data_for_project(
project['name'])
if not st_data:
# It's possible this is a project listed by its
# service-type
st_data = service_types.get_service_data(st)
if not st_data:
logger.error(
'did not find %s in Service Types Authority',
project['name'],
)
fail = True
elif st != st_data['service_type']:
logger.error(
'expected service_type %r for %s but got %r',
st_data['service_type'],
project['name'],
st,
)
fail = True
# client projects must have a description
project_type = project.get('type')
if (project_type in ['cloud-client', 'service-client']
and not project.get('description')):
logger.error(
'client project %s has no description',
project['name'],
)
fail = True
# If the project claims to have a separately published guide
# of some sort, look for it before allowing the flag to stand.
check_links_this_project = (deliverable_name in projects_to_check
or not projects_to_check)
if check_links_this_project and not skip_links:
for url_info in _URLS:
flag_val = project.get(url_info.flag_name,
url_info.default)
if ((not flag_val) and url_info.types
and project_type not in url_info.types):
# This type of project isn't expected to have
# this type of link, so if we are not
# explicitly told to check for it don't.
continue
try:
url = url_info.template.format(series=series,
**project)
except KeyError:
# The project data does not include a field needed
# to build the URL (typically the
# service_type). Ignore this URL, unless the flag
# is set.
if flag_val:
raise
continue
# Only try to fetch the URL if we're going to do
# something with the result.
if flag_val or check_all_links:
logger.info('%s:%s looking for %s', series,
project['name'], url)
links_to_check.append((url, project['name'],
url_info.flag_name, flag_val))
if links_to_check:
logger.info('checking %s links from %s...', len(links_to_check),
filename)
pool = multiprocessing.pool.ThreadPool()
results = pool.map(_check_url, links_to_check)
for url, project_name, flag, flag_val, exists, status in results:
if flag_val and not exists:
logger.error(
'%s set for %s but %s does not exist (%s)',
flag,
project_name,
url,
status,
)
fail = True
elif (not flag_val) and check_all_links and exists:
msg = '{} not set for {} but {} does exist'.format(
flag, project_name, url)
logger.warning(msg)
if strict:
raise RuntimeError(msg)
if fail:
raise ValueError('invalid input in %s' % filename)
project_data[series] = data
return project_data
def test_validate_upgrade(self):
body = self.upgrade
schema = self.controller.get_schema('action', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
import json
import pkgutil
import re
try:
# Python 2
from collections import Mapping, Sequence
except ImportError:
# Python 3
from collections.abc import Mapping, Sequence
import jsonschema
from funcsigs import signature
from .exceptions import InvalidParams, InvalidRequest, MethodNotFound
_JSON_VALIDATOR = jsonschema.Draft4Validator(
json.loads(pkgutil.get_data(__name__, 'request-schema.json').decode()))
def convert_camel_case(name):
"""Convert a camelCase string to under_score"""
string = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', string).lower()
def convert_camel_case_keys(original_dict):
"""Converts all keys of a dict from camelCase to under_score, recursively"""
new_dict = dict()
for key, val in original_dict.items():
if isinstance(val, dict):
# Recurse
new_dict[convert_camel_case(key)] = convert_camel_case_keys(val)
def parse_config(configfile, refpath=None, testpath=None):
print('\n=====CONFIGURATION=====')
# check inputs
if configfile and not os.path.isfile(configfile):
raise IOError("Configuration file does not exist")
if refpath and not os.path.isdir(refpath):
raise IOError('"refpath" not a valid folder <{}>'.format(refpath))
if testpath and not os.path.isdir(testpath):
raise IOError('"testpath" not a valid folder <{}>'.format(testpath))
# create schema validator object (& check schema itself)
schema = json.loads(
pkg_resources.resource_string(resource_package,
'config_schema.json').decode('utf-8'))
validator = jsonschema.Draft4Validator(schema)
validator.check_schema(schema)
# load user configuration
print("\nReading configuration from <{}>".format(configfile))
# JSON parsing
if configfile.endswith(('.json', '.JSON')):
# open & read JSON file
with open(configfile, 'r') as fid:
config = json.load(fid)
# CONFIG parsing
elif configfile.endswith(('.config', '.CONFIG')):
# setup config parser
parser = configparser.ConfigParser()
parser.optionxform = str # maintain case-sensitive items
# read entire configuration file into dict
if len(parser.read(configfile)) == 0:
raise IOError("Unable to read selected .config file")
config = {s: dict(parser.items(s)) for s in parser.sections()}
# special section/item parsing
s = 'INPUT.REF'
i = 'CLSMatchValue'
config[s][i] = ast.literal_eval(config[s][i])
s = 'INPUT.TEST'
i = 'CLSMatchValue'
if i in config[s]: # Optional Field
config[s][i] = ast.literal_eval(config[s][i])
else:
config[s][i] = config['INPUT.REF'][i]
# bool(config[s][i]) does not interpret 'true'/'false' strings
s = 'OPTIONS'
i = 'QuantizeHeight'
config[s][i] = parser.getboolean(s, i)
s = 'PLOTS'
i = 'ShowPlots'
config[s][i] = parser.getboolean(s, i)
s = 'PLOTS'
i = 'SavePlots'
config[s][i] = parser.getboolean(s, i)
s = 'MATERIALS.REF'
i = 'MaterialNames'
config[s][i] = config[s][i].split(',')
s = 'MATERIALS.REF'
i = 'MaterialIndicesToIgnore'
config[s][i] = [int(v) for v in config[s][i].split(',')]
# unrecognized config file type
else:
raise IOError('Unrecognized configuration file')
# locate files for each "xxxFilename" configuration parameter
# this makes use of "refpath" and "testpath" arguments for relative filenames
# we do this before validation to ensure required files are located
for item in [('INPUT.REF', refpath), ('INPUT.TEST', testpath)]:
sec = item[0]
path = item[1]
print('\nPROCESSING "{}" FILES'.format(sec))
config[sec] = findfiles(config[sec], path)
# validate final configuration against schema
try:
validator.validate(config)
print('\nCONFIGURATION VALIDATED')
except jsonschema.exceptions.ValidationError:
print('\n*****INVALID CONFIGURATION FILE*****\n')
for error in sorted(validator.iter_errors(config), key=str):
print('ERROR: {}\n'.format(error))
raise jsonschema.exceptions.ValidationError('validation error')
# for easier explotation, ensure some configuration options are tuple/list
opts = (('INPUT.TEST', 'CLSMatchValue'), ('INPUT.REF', 'CLSMatchValue'),
('MATERIALS.REF', 'MaterialIndicesToIgnore'))
for opt in opts:
s = opt[0]
i = opt[1]
try:
_ = (v for v in config[s][i])
except:
config[s][i] = [config[s][i]]
# print final configuration
print('\nFINAL CONFIGURATION')
print(json.dumps(config, indent=2))
# cleanup
return config
import enhancements
from http import HttpConnection
from opsgenie import OpsGenieAlerter
import ruletypes
from util import dt_to_ts
from util import dt_to_ts_with_format
from util import dt_to_unix
from util import dt_to_unixms
from util import EAException
from util import ts_to_dt
from util import ts_to_dt_with_format
from util import unix_to_dt
from util import unixms_to_dt
# schema for rule yaml
rule_schema = jsonschema.Draft4Validator(
yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml'))))
# Required global (config.yaml) and local (rule.yaml) configuration options
required_globals = frozenset([
'run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index',
'buffer_time'
])
required_locals = frozenset(['alert', 'type', 'name', 'index'])
# Settings that can be derived from ENV variables
env_settings = {
'ES_USE_SSL': 'use_ssl',
'ES_PASSWORD': '******',
'ES_USERNAME': '******',
'ES_HOST': 'es_host',
'ES_PORT': 'es_port'
def check_scoring(json_conf):
"""
:param json_conf: configuration dictionary to check.
:type json_conf: dict
Function to check the "scoring" section of the configuration.
Each scoring function will be checked for:
- validity of the expression (it can be interpreted by Mikado)
- validity of the parameter (it is a valid Metric)
:return: json_conf
:rtype dict
"""
with io.TextIOWrapper(resource_stream(__name__,
"scoring_blueprint.json")) as schema:
scoring_schema = json.load(schema)
parameters_found = set()
parameters_not_found = []
double_parameters = []
invalid_filter = set()
available_metrics = Transcript.get_available_metrics()
if "scoring" not in json_conf or len(json_conf["scoring"].keys()) == 0:
raise InvalidJson("No parameters specified for scoring!")
validator = extend_with_default(jsonschema.Draft4Validator)
for parameter in json_conf["scoring"]:
if parameter not in available_metrics:
parameters_not_found.append(parameter)
if parameter in parameters_found:
double_parameters.append(parameter)
if not jsonschema.Draft4Validator(scoring_schema).is_valid(
json_conf["scoring"][parameter]):
errors = [
str(_) for _ in list(
jsonschema.Draft4Validator(scoring_schema).iter_errors(
json_conf["scoring"][parameter]))
]
raise InvalidJson("Invalid scoring for {}:\n{}".format(
parameter, "\n".join(errors)))
try:
validator(scoring_schema).validate(json_conf["scoring"][parameter])
except Exception as err:
raise ValueError(parameter, err)
if json_conf["scoring"][parameter]["rescaling"] == "target":
if "value" not in json_conf["scoring"][parameter]:
message = """Target rescaling requested for {0} but no target value specified.
Please specify it with the \"value\" keyword.\n{1}"""
message = message.format(parameter,
json_conf["scoring"][parameter])
raise UnrecognizedRescaler(message)
if len(parameters_not_found) > 0 or len(double_parameters) > 0 or len(
invalid_filter) > 0:
err_message = ''
if len(parameters_not_found) > 0:
err_message = """The following parameters, present in the JSON file,
are not available!\n\t{0}\n""".format(
"\n\t".join(parameters_not_found))
if len(double_parameters) > 0:
err_message += """The following parameters have been specified more than once,
please correct:\n\t{0}""".format("\n\t".join(
list(double_parameters)))
if len(invalid_filter) > 0:
err_message += """The following parameters have an invalid filter,
please correct:
\t{0}""".format("\n\t".join(list(invalid_filter)))
raise InvalidJson(err_message)
return json_conf
def __init__(self):
self._schema = jsonschema.Draft4Validator(CONFIG_SCHEMA)
def check_requirements(json_conf, require_schema, index):
"""
Function to check the "requirements" section of the configuration.
Each filtering function will be checked for:
- validity of the expression (it can be interpreted by Mikado)
- validity of the parameter (it is a valid Metric)
:param json_conf: configuration dictionary to check.
:type json_conf: dict
:param require_schema: the requirements section of the JSON schema.
:type require_schema: dict
:return: json_conf
:rtype dict
"""
# Check that the parameters are valid
parameters_not_found = []
available_metrics = Transcript.get_available_metrics()
if "parameters" not in json_conf[index]:
raise InvalidJson(
"The {} field must have a \"parameters\" subfield!".format(index))
for key in json_conf[index]["parameters"]:
key_name = key.split(".")[0]
if key_name not in available_metrics:
parameters_not_found.append(key_name)
continue
if not jsonschema.Draft4Validator(
require_schema["definitions"]["parameter"]).is_valid(
json_conf[index]["parameters"][key]):
errors = list(
jsonschema.Draft4Validator(require_schema).iter_errors(
json_conf[index]["parameters"][key]))
raise InvalidJson("Invalid parameter for {0} in {1}: \n{2}".format(
key, index, errors))
json_conf[index]["parameters"][key]["name"] = key_name
if len(parameters_not_found) > 0:
raise InvalidJson(
"The following parameters, selected for filtering, are invalid:\n\t{0}"
.format("\n\t".join(parameters_not_found)))
# Create automatically a filtering expression
if "expression" not in json_conf[index]:
json_conf[index]["expression"] = " and ".join(
list(json_conf[index]["parameters"].keys()))
keys = json_conf[index]["parameters"].keys()
newexpr = json_conf[index]["expression"][:]
else:
if not jsonschema.Draft4Validator(
require_schema["definitions"]["expression"]).is_valid(
json_conf[index]["expression"]):
raise InvalidJson("Invalid expression field")
expr = " ".join(json_conf[index]["expression"])
newexpr = expr[:]
keys = list(key for key in re.findall("([^ ()]+)", expr)
if key not in ("and", "or", "not", "xor"))
diff_params = set.difference(
set(keys), set(json_conf[index]["parameters"].keys()))
if len(diff_params) > 0:
raise InvalidJson(
"Expression and required parameters mismatch:\n\t{0}".format(
"\n\t".join(list(diff_params))))
for key in keys: # Create the final expression
newexpr = re.sub(key, "evaluated[\"{0}\"]".format(key), newexpr)
# Test the expression
try:
compile(newexpr, "<json>", "eval")
except SyntaxError:
raise InvalidJson("Invalid expression for {}:\n{}".format(
index, newexpr))
json_conf[index]["expression"] = newexpr
return json_conf
def test_validate_create_complete(self):
body = {"backup": {"instance": self.uuid, "name": "testback-backup"}}
schema = self.controller.get_schema('create', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
import jsonschema
import pkg_resources
try:
import simplejson as json
except ImportError:
import json
_FORM_SCHEMA_JSON = json.loads(
pkg_resources.resource_string(
"slivka",
"data/config/FormDescriptionSchema.json"
).decode()
)
jsonschema.Draft4Validator.check_schema(_FORM_SCHEMA_JSON)
FORM_VALIDATOR = jsonschema.Draft4Validator(_FORM_SCHEMA_JSON)
_COMMAND_SCHEMA_JSON = json.loads(
pkg_resources.resource_string(
"slivka",
"data/config/ConfDescriptionSchema.json"
).decode()
)
jsonschema.Draft4Validator.check_schema(_COMMAND_SCHEMA_JSON)
COMMAND_VALIDATOR = jsonschema.Draft4Validator(_COMMAND_SCHEMA_JSON)
class Singleton(type):
__instances = {}
def __call__(cls, *args, **kwargs):
class Server(with_metaclass(ABCMeta, object)):
"""Protocol-agnostic class representing the remote server. Subclasses should
inherit and override ``_send_message``.
:param endpoint: The server address.
"""
# Request and response logs
__request_log = logging.getLogger(__name__ + '.request')
__response_log = logging.getLogger(__name__ + '.response')
#: Validate the response message
__validator = jsonschema.Draft4Validator(
json.loads(
pkgutil.get_data(__name__,
'response-schema.json').decode('utf-8')))
def __init__(self, endpoint):
#: Holds the server address
self.endpoint = endpoint
def _log_request(self, request, extra=None):
"""Log the JSON-RPC request before sending. Should be called by
subclasses in :meth:`_send_message`, before sending.
:param request: The JSON-RPC request string.
:param extra: A dict of extra fields that may be logged.
"""
if extra is None:
extra = {}
# Add endpoint to list of info to include in log message
extra.update({'endpoint': self.endpoint})
_log(self.__request_log,
'info',
request,
fmt='--> %(message)s',
extra=extra)
def _log_response(self, response, extra=None):
"""Log the JSON-RPC response after sending. Should be called by
subclasses in :meth:`_send_message`, after receiving the response.
:param response: The JSON-RPC response string.
:param extra: A dict of extra fields that may be logged.
"""
if extra is None:
extra = {}
# Add the endpoint to the log entry
extra.update({'endpoint': self.endpoint})
# Clean up the response for logging
response = response.replace("\n", '').replace(' ', ' ') \
.replace('{ ', '{')
_log(self.__response_log,
'info',
response,
fmt='<-- %(message)s',
extra=extra)
def _process_response(self, response):
"""Processes the response and returns the 'result' portion if present.
:param response: The JSON-RPC response string to process.
:return: The response string, or None
"""
if response:
if isinstance(response, basestring):
# Attempt to parse the response
try:
response = json.loads(response)
except ValueError:
raise exceptions.ParseResponseError()
# Validate the response against the Response schema (raises
# jsonschema.ValidationError if invalid)
if config.validate:
self.__validator.validate(response)
if isinstance(response, list):
# For now, just return the whole response
return response
else:
# If the response was "error", raise to ensure it's handled
if 'error' in response:
raise exceptions.ReceivedErrorResponse(
response['error'].get('code'),
response['error'].get('message'),
response['error'].get('data'))
# else
return response.get('result')
# No response was given
return None
@abstractmethod
def _send_message(self, request, **kwargs):
"""Used internally - send the request to the server. Override this
method in the protocol-specific subclasses. Be sure to log both the
request and response, and return the response.
:param request: A JSON-RPC request, in dict format.
:returns: The response (a string for requests, None for notifications).
"""
def send(self, request, **kwargs):
"""Send a request, passing the whole JSON-RPC `request object
<http://www.jsonrpc.org/specification#request_object>`_.
>>> server.send({'jsonrpc': '2.0', 'method': 'ping', 'id': 1})
--> {"jsonrpc": "2.0", "method": "ping", "id": 1}
<-- {"jsonrpc": "2.0", "result": "pong", "id": 1}
'pong'
:param request: The JSON-RPC request.
:type request: string or a JSON serializable object
:param kwargs: For HTTPServer, these are passed on to
`requests.Session.send()
<http://docs.python-requests.org/en/master/api/#requests.Session.send>`_.
:returns: The payload (i.e. the ``result`` part of the response, or
``None`` in the case of a Notification).
:rtype: A `JSON-decoded object
<https://docs.python.org/library/json.html#json-to-py-table>`_.
:raises ParseResponseError:
The response was not valid JSON.
:raises ValidationError:
The response was valid JSON, but not valid JSON-RPC.
:raises ReceivedErrorResponse:
The server responded with a JSON-RPC `error object
<http://www.jsonrpc.org/specification#error_object>`_.
"""
# Convert request to a string
if not isinstance(request, basestring):
request = json.dumps(request)
# Call internal method to transport the message
response = self._send_message(request, **kwargs)
return self._process_response(response)
# Alternate ways to send a request -----------
def notify(self, method_name, *args, **kwargs):
"""Send a JSON-RPC request, without expecting a response.
:param method_name: The remote procedure's method name.
:param args: Positional arguments passed to the remote procedure.
:param kwargs: Keyword arguments passed to the remote procedure.
:return: The payload (i.e. the ``result`` part of the response).
"""
return self.send(Notification(method_name, *args, **kwargs))
def request(self, method_name, *args, **kwargs):
"""Send a request, by passing the method and arguments. This is the main
public method.
>>> server.request('cat', name='Mittens')
--> {"jsonrpc": "2.0", "method": "cat", "params": {"name": "Mittens"}, "id": 1}
<-- {"jsonrpc": "2.0", "result": "meow", "id": 1}
'meow'
:param method_name: The remote procedure's method name.
:param args: Positional arguments passed to the remote procedure.
:param kwargs: Keyword arguments passed to the remote procedure.
:return: The payload (i.e. the ``result`` part of the response).
"""
return self.send(Request(method_name, *args, **kwargs))
def __getattr__(self, name):
"""This gives us an alternate way to make a request::
>>> server.cube(3)
27
That's the same as saying ``server.request('cube', 3)``.
Technique is explained here: http://code.activestate.com/recipes/307618/
"""
def attr_handler(*args, **kwargs):
"""Call self.request from here"""
return self.request(name, *args, **kwargs)
return attr_handler
def match(schema: dict, response: Any) -> List[dict]:
validator = jsonschema.Draft4Validator(schema)
errors = list(validator.iter_errors(response))
return [dict(value=list(e.path), message=e.message) for e in errors]
def _validate_jsonschema(instance, schema):
validator = jsonschema.Draft4Validator(schema)
errors = list(validator.iter_errors(instance))
if len(errors) != 0:
sys.exit("\tERROR\n\nValidation error: {}".format(errors))
ERRORS_SCHEMA = os.path.join(cur_dir, '.schemacache', 'errors', 'payload.json')
TRANSACTIONS_SCHEMA = os.path.join(cur_dir, '.schemacache', 'transactions',
'payload.json')
assert (os.path.exists(ERRORS_SCHEMA) and os.path.exists(TRANSACTIONS_SCHEMA)), \
'JSON Schema files not found. Run "make update-json-schema to download'
with codecs.open(ERRORS_SCHEMA, encoding='utf8') as errors_json, \
codecs.open(TRANSACTIONS_SCHEMA, encoding='utf8') as transactions_json:
VALIDATORS = {
'/v1/errors':
jsonschema.Draft4Validator(
json.load(errors_json),
resolver=jsonschema.RefResolver(
base_uri='file:' + pathname2url(ERRORS_SCHEMA),
referrer='file:' + pathname2url(ERRORS_SCHEMA),
)),
'/v1/transactions':
jsonschema.Draft4Validator(
json.load(transactions_json),
resolver=jsonschema.RefResolver(
base_uri='file:' + pathname2url(TRANSACTIONS_SCHEMA),
referrer='file:' + pathname2url(TRANSACTIONS_SCHEMA),
))
}
class ValidatingWSGIApp(ContentServer):
def __init__(self, **kwargs):
super(ValidatingWSGIApp, self).__init__(**kwargs)
def validate_armada_document(document):
"""Validates a document ingested by Armada by subjecting it to JSON schema
validation.
:param dict dictionary: The document to validate.
:returns: A tuple of (bool, list[dict]) where the first value
indicates whether the validation succeeded or failed and
the second value is the validation details with a minimum
keyset of (message(str), error(bool))
:rtype: tuple.
:raises TypeError: If ``document`` is not of type ``dict``.
"""
if not isinstance(document, dict):
raise TypeError('The provided input "%s" must be a dictionary.' %
document)
schema = document.get('schema', '<missing>')
document_name = document.get('metadata', {}).get('name', None)
details = []
LOG.debug('Validating document [%s] %s', schema, document_name)
if schema in SCHEMAS:
try:
validator = jsonschema.Draft4Validator(SCHEMAS[schema])
for error in validator.iter_errors(document.get('data')):
error_message = "Invalid document [%s] %s: %s." % \
(schema, document_name, error.message)
vmsg = ValidationMessage(message=error_message,
error=True,
name='ARM100',
level='Error',
schema=schema,
doc_name=document_name)
LOG.info('ValidationMessage: %s', vmsg.get_output_json())
details.append(vmsg.get_output())
except jsonschema.SchemaError as e:
error_message = ('The built-in Armada JSON schema %s is invalid. '
'Details: %s.' % (e.schema, e.message))
vmsg = ValidationMessage(message=error_message,
error=True,
name='ARM000',
level='Error',
diagnostic='Armada is misconfigured.')
LOG.error('ValidationMessage: %s', vmsg.get_output_json())
details.append(vmsg.get_output())
else:
vmsg = ValidationMessage(message='Unsupported document type.',
error=False,
name='ARM002',
level='Warning',
schema=schema,
doc_name=document_name,
diagnostic='Please ensure document is one of '
'the following schema types: %s' %
list(SCHEMAS.keys()))
LOG.info('Unsupported document type, ignoring %s.', schema)
LOG.debug('ValidationMessage: %s', vmsg.get_output_json())
# Validation API doesn't care about this type of message, don't send
if len([x for x in details if x.get('error', False)]) > 0:
return False, details
return True, details
def validate_training_request(request):
"""Validate a JSON-formatted training request."""
return jsonschema.Draft4Validator(schema).is_valid(request)
