def validate(self):
"""validates the JSON
Raises:
InvalidServerSettings: bad entry in the JSON
"""
try:
humanize.validate_with_humanized_errors(self.settings, self.schema)
except voluptuous.Error as error:
self.log.error(error)
raise InvalidSettings("bad settings {}".format(error))
return
def validateItems(**kwargs):
for k in kwargs.keys():
if k not in __validator__:
print("ERROR! got unrecognized key: {}".format(k))
return 0
S = Schema({k: v for k, v in __validator__.items()}, extra=PREVENT_EXTRA)
return validate_with_humanized_errors(kwargs, S)
def assemble_args(self, **options):
options = validate_with_humanized_errors(options, cudaDeconSchema)
arglist = []
for o in options:
# convert LLSpy variable naming conventions to cudaDeconv names
# TODO: consider uniying everything to cudaDeconv?
optname = o
convert_name = {
'nIters': 'RL',
}
if optname in convert_name:
optname = convert_name[optname]
# assemble the argument list
if self.has_option_longname(optname):
# expand listed items like --MIP 0 0 0
if optname in ('MIP', 'rMIP', 'crop'):
arglist.append('--' + optname)
[arglist.append(str(i)) for i in options[o]]
# booleans only get a single flag
elif isinstance(options[o], bool):
if options[o]:
arglist.extend(['--' + optname])
# otherwise just add the argument
else:
arglist.extend(['--' + optname, str(options[o])])
else:
logger.warn(
'Warning: option not recognized, ignoring: {}'.format(o))
return arglist
def post(self):
try:
vhum.validate_with_humanized_errors(request.json,
self.LOGIN_VALIDATOR)
except verr.Error as invalid:
raise BadRequest(str(invalid))
try:
user = UserAccount.query.filter_by(
email=request.json.get('email')).one()
except NoResultFound:
raise Unauthorized('Email or password incorrect')
if not user.verify_password(request.json.get('password')):
raise Unauthorized('Email or password incorrect')
token = user.generate_auth_token()
return {'access_token': token.decode('ascii')}
def __init__(self, *args, **kwargs):
"""
Service configuration class constructor
"""
super(ServiceConfiguration, self).__init__()
self.dictionary = validate_with_humanized_errors(
kwargs, self.VALIDATION_SCHEMA)
def load(cls, src):
articles = yaml.safe_load(src)
# validate
validated = validate_with_humanized_errors(articles, ARTICLES)
# instantiate
rules = [Article.from_db(a) for a in validated]
return cls(rules)
def validate_parts(self, k, parts):
## add for submeasure by datang 11/9/2019
##if (len(parts)==1 and parts[0].keys[0]=='submeasure'):
## return parts
parts = validate_with_humanized_errors(
parts, response_type.response_parts_schema)
response_type.validate_parts(parts)
return parts
def get_mozilla_office_ips():
try:
mozilla_ip_ranges_file = Path('/var/hg/moz-ip-ranges.txt')
bloxtool_config_file = Path('/etc/mercurial/bloxtool.ini')
bloxtool_command = [
'/var/hg/venv_tools/bin/bloxtool', 'network', 'search',
'attribute', 'subnet-purpose', 'value', 'nat-pool',
'--format=json', f'--config={bloxtool_config_file}'
]
bloxtool_json_schema = Schema([{
'comment': str,
'_ref': str,
'network': is_ip_address_network,
'network_view': str,
}],
extra=False,
required=True)
# Get raw string output and convert to Python dict
process_output = subprocess.run(bloxtool_command,
check=True,
encoding='utf-8',
stdout=subprocess.PIPE).stdout
output_as_dict = json.loads(process_output)
# Verify dict schema
validate_with_humanized_errors(output_as_dict, bloxtool_json_schema)
write_to_file_atomically(
mozilla_ip_ranges_file,
'\n'.join(i['network'] for i in output_as_dict))
except subprocess.CalledProcessError as cpe:
sys.exit('An error occurred while executing the bloxtool command.')
except json.JSONDecodeError as jde:
sys.exit('An error occurred parsing the bloxtool output as JSON.')
except VoluptuousInvalid as vi:
sys.exit(
'The JSON data from bloxtool does not match the required schema.')
def post(self):
try:
vhum.validate_with_humanized_errors(request.json,
self.REGISTER_VALIDATOR)
except verr.Error as invalid:
raise BadRequest(str(invalid))
email = request.json['email']
user_with_same_email = UserAccount.query.filter(
UserAccount.email == email).one_or_none()
if user_with_same_email:
raise Conflict(f"Email '{email}' is already taken.")
password = request.json.pop('password')
new_user = UserAccount(**request.json)
new_user.hash_password(password)
db.session.add(new_user)
db.session.commit()
return new_user, 201
def on_get(self, req, resp, word):
"""
Return http response
Get method for get tweets of word
"""
data = req.params
validate_with_humanized_errors(data, SearchTweets.validator)
if 'limit' in data and data.get('limit', '').isdigit():
limit = int(data.get('limit'))
elif 'limit' in data and not data.get('limit', '').isdigit():
raise falcon.HTTPBadRequest('Invalid parameter in endpoint')
else:
limit = self.default
try:
tweet_word = TwitterSearch(word)
tweets = tweet_word.get_tweets(limit)
resp.body = tweets
resp.status = falcon.HTTP_200
except Exception as error:
resp.status = falcon.HTTP_404
def localParams(*args, **kwargs):
""" returns a validated dict of processing parameters
with defaults filled in when not supplied, that ALSO
contains parameters to a specific LLSdir instance.
returned by llspy.llsdir.localParams()
"""
if len(args) == 1 and isinstance(args[0], dict):
kwargs = args[0]
S = validate_with_humanized_errors(kwargs, __localSchema__)
return dotdict(S)
def on_get(self, req, resp, user):
"""
Return http response
Get method for get tweets of user
"""
data = req.params
validate_with_humanized_errors(data, UserTweets.validator)
if 'limit' in data and data.get('limit', '').isdigit():
limit = int(data.get('limit'))
elif 'limit' in data and not data.get('limit', '').isdigit():
raise falcon.HTTPBadRequest('Invalid parameter in endpoint')
else:
limit = self.default
try:
tweet_user = TwitterUser(user)
tweets = tweet_user.get_tweets(limit)
resp.body = tweets
resp.status = falcon.HTTP_200
except Exception as error:
resp.body = json.dumps({'Error': str(error)})
resp.status = falcon.HTTP_404
def procParams(*args, **kwargs):
""" returns a validated dict of processing parameters
with defaults filled in when not supplied.
>>> P = procParams() # get default parameters
>>> P = procParams(nIters=7, tRange=range(0,10))
# check validitity of parameter name
>>> 'regMode' in procParams()
"""
# accept a single dict as well as expanded options
if len(args) == 1 and isinstance(args[0], dict):
kwargs = args[0]
S = validate_with_humanized_errors(kwargs, __schema__)
if S['nIters'] > 0 and S['otfDir'] is None:
raise ParametersError('oftDir cannot be type None with nIters > 0')
return dotdict(S)
def validate(schema, data):
# TODO - print valid keys when not allowed key is used
try:
return validate_with_humanized_errors(data, Schema(schema))
except Exception as e:
raise e
def get_mozilla_office_ips():
'''Entry point for the Mozilla office IP scraper
Calls out to bloxtool to obtain Mozilla network information
in JSON format. Validates the JSON against a known schema and
atomically re-writes a file with the CIDR representations of
Mozilla office IP address spaces.
'''
try:
mozilla_ip_ranges_file = Path('/var/hg/moz-ip-ranges.txt')
bloxtool_config_file = Path('/etc/mercurial/bloxtool.ini')
bloxtool_command = [
'/var/hg/venv_tools/bin/bloxtool', 'network', 'search',
'attribute', 'subnet-purpose', 'value', 'nat-pool',
'--format=json', f'--config={bloxtool_config_file}'
]
bloxtool_json_schema = Schema([{
'comment': str,
'_ref': str,
'network': is_ip_address_network,
'network_view': str,
}],
extra=False,
required=True)
# Get raw string output and convert to Python dict
process_output = subprocess.run(bloxtool_command,
check=True,
encoding='utf-8',
stdout=subprocess.PIPE).stdout
output_as_dict = json.loads(process_output)
# Verify dict schema
validate_with_humanized_errors(output_as_dict, bloxtool_json_schema)
write_to_file_atomically(
mozilla_ip_ranges_file,
'\n'.join(i['network'] for i in output_as_dict))
except subprocess.CalledProcessError as cpe:
logger.exception(
'An error occurred while executing the bloxtool command: exit code %s'
% cpe.returncode)
logger.exception('STDOUT: %s' % cpe.stdout)
logger.exception('STDERR: %s' % cpe.stderr)
sys.exit(1)
except json.JSONDecodeError as jde:
logger.exception(
'An error occurred parsing the bloxtool output as JSON: %s' %
jde.msg)
sys.exit(1)
except VoluptuousInvalid as vi:
logger.exception(
'The JSON data from bloxtool does not match the required schema.')
logger.exception('Error message: %s' % vi.msg)
logger.exception('Error path: %s' % vi.path)
logger.exception('Exception message: %s' % vi.error_message)
sys.exit(1)
def get_aws_ips():
'''Entry point for the AWS IP address scraper
Downloads the AWS IP ranges JSON document from Amazon and verifies against a
known schema. Atomically rewrites a file with the CIDR representations of
AWS IP address spaces.
'''
try:
# Grab the new data from Amazon
amazon_ip_ranges_file = Path('/var/hg/aws-ip-ranges.json')
ip_ranges_response = requests.get(
'https://ip-ranges.amazonaws.com/ip-ranges.json')
# Ensure 200 OK response code
if ip_ranges_response.status_code != 200:
sys.exit('HTTP response from Amazon was not 200 OK')
# Sanity check: ensure the file is an appropriate size
if len(ip_ranges_response.content) < 88000:
sys.exit(
'The retrieved AWS JSON document is smaller than the minimum allowable file size'
)
# JSON Schema for the Amazon IP Ranges JSON document
amazon_json_schema = Schema(
{
'syncToken':
str,
'createDate':
str,
'ipv6_prefixes': [
dict
], # If IPv6 is supported in the future, this will need to be defined
# The prefixes field must meet both requirements:
# 1. There must be at least one entry for each region containing CI and S3 bundles
# 2. Must be a list of dicts that fit the schema below
'prefixes':
All(all_required_regions_exist, [
{
'ip_prefix': is_ip_address_network,
'region': str,
'service': str,
},
]),
},
extra=False,
required=True)
# Validate dict schema
output_as_dict = ip_ranges_response.json()
validate_with_humanized_errors(output_as_dict, amazon_json_schema)
# Sanity check: ensure the syncToken indicates an IP space change has been made
# since the last recorded change. Only check if a file exists, in case of new deployments
if amazon_ip_ranges_file.is_file():
file_bytes = amazon_ip_ranges_file.read_bytes()
existing_document_as_dict = json.loads(file_bytes)
file_diff = diff(existing_document_as_dict,
output_as_dict,
context=0)
# Exit if the file contents are the same or the syncToken has not changed
if not file_diff or int(output_as_dict['syncToken']) <= int(
existing_document_as_dict['syncToken']):
sys.exit()
else:
existing_document_as_dict = {
} # No existing document means whole file is the diff
file_diff = diff(existing_document_as_dict,
output_as_dict,
context=0)
write_to_file_atomically(amazon_ip_ranges_file,
json.dumps(output_as_dict))
# Print the diff for collection as systemd unit output
logger.info('AWS IP ranges document has been updated')
logger.info(file_diff)
except subprocess.CalledProcessError as cpe:
logger.exception(
'An error occurred when notifying about changes to the file: exit code %s'
% cpe.returncode)
logger.exception('STDOUT: %s' % cpe.stdout)
logger.exception('STDERR: %s' % cpe.stderr)
sys.exit(1)
except json.JSONDecodeError as jde:
logger.exception(
'An error occurred parsing the data retrieved from Amazon as JSON: %s'
% jde.msg)
sys.exit(1)
except VoluptuousInvalid as vi:
logger.exception(
'The JSON data from Amazon does not match the required schema.')
logger.exception('Error message: %s' % vi.msg)
logger.exception('Error path: %s' % vi.path)
logger.exception('Exception message: %s' % vi.error_message)
sys.exit(1)
def validate_structure(self, k, s):
return validate_with_humanized_errors(s, Survey._structure_schema)
def verify(data, *validators):
assert validate_with_humanized_errors(data, Schema(All(*validators))) == data
ATTRIBUTE_SCHEMA = All(Schema({str: [BROADCAST_SCHEMA]}), validate_broadcast)
MODULE_INFO_SCHEMA = All(
Schema({
Required('code'): Coerce(int),
Required('protocol'): All(Lower, Any('any', Coerce(int))),
Required('software_versions'): All([Coerce(int)]),
Required('description'): Coerce(str),
Required('attributes'): ATTRIBUTE_SCHEMA,
}))
SCHEMA = All(Schema({
str: MODULE_INFO_SCHEMA,
}))
validate_with_humanized_errors(data=module_data, schema=SCHEMA)
module_info = SCHEMA(module_data)
class ModuleFactory:
"""This is a ModuleFactory class to create specific module class based on discovered data."""
factories = {}
@staticmethod
def create_module(can_id, code, protocol, software_version):
"""Create module object based on proivded data.
Args:
can_id (int): CAN ID
code (int): Module Code Number
def verify(data, *validators):
assert validate_with_humanized_errors(data,
Schema(All(*validators))) == data
'network_view': str,
}],
extra=False,
required=True)
if __name__ == '__main__':
try:
# Get raw string output and convert to Python dict
process_output = subprocess.run(COMMAND,
check=True,
encoding='utf-8',
stdout=subprocess.PIPE).stdout
output_as_dict = json.loads(process_output)
# Verify dict schema
validate_with_humanized_errors(output_as_dict, JSON_SCHEMA)
# Write data to a temp file, atomically rewrite the the IP ranges file
temp_file_path = IP_RANGES_FILE.with_suffix('.tmp')
with temp_file_path.open(mode='w') as temp_file:
temp_file.write('\n'.join(i['network'] for i in output_as_dict))
temp_file_path.rename(IP_RANGES_FILE)
except subprocess.CalledProcessError as cpe:
sys.exit('An error occurred while executing the bloxtool command.')
except json.JSONDecodeError as jde:
sys.exit('An error occurred parsing the bloxtool output as JSON.')
except VoluptuousInvalid as vi:
def validate_dict(cls, dct):
return validate_with_humanized_errors(dct, cls.schema)
def validate_settings(self, premodel_settings, session_type):
'''
This method validates the premodel settings for the 'data_new',
'data_append', 'model_generate', or 'model_predict' sessions.
Note: This method does not validate the associated 'file upload(s)',
which is the responsibility of the mongodb query process.
'''
# local variables
model_type = current_app.config.get('MODEL_TYPE')
dataset_type = current_app.config.get('DATASET_TYPE')
sv_kernel_type = current_app.config.get('SV_KERNEL_TYPE')
# validation on 'data_new', 'data_append' session
if session_type in ['data_new', 'data_append']:
if premodel_settings[
'stream'] == 'True' or session_type == 'data_new':
schema = Schema({
Required('collection'):
All(unicode, Length(min=1)),
Required('dataset_type'):
In(dataset_type),
Required('model_type'):
In(model_type),
Required('session_type'):
Any('data_new', 'data_append'),
Required('session_name'):
All(unicode, Length(min=1)),
Optional('stream'):
Any('True', 'False'),
})
else:
schema = Schema({
Required('collection'):
All(unicode, Length(min=1)),
Required('dataset_type'):
In(dataset_type),
Required('model_type'):
In(model_type),
Required('session_type'):
Any('data_new', 'data_append'),
Optional('stream'):
Any('True', 'False'),
})
# validation on 'model_generate' session
if session_type == 'model_generate':
schema = Schema({
Required('collection'):
All(unicode, Length(min=1)),
Required('model_type'):
In(model_type),
Required('session_type'):
'model_generate',
Optional('stream'):
Any('True', 'False'),
Required('sv_kernel_type'):
In(sv_kernel_type),
Optional('gamma'):
Any(Coerce(int), Coerce(float)),
Optional('penalty'):
Any(Coerce(int), Coerce(float)),
})
# validation on 'model_predict' session
elif session_type == 'model_predict':
schema = Schema({
Required('collection'):
All(unicode, Length(min=1)),
Optional('stream'):
Any('True', 'False'),
Required('prediction_input[]'): [
Any(Coerce(int), Coerce(float)),
],
Required('session_type'):
'model_predict',
})
try:
validate_with_humanized_errors(premodel_settings, schema)
except Exception, error:
split_error = str(error).splitlines()
self.list_error.append(split_error)
return split_error
def validate_response_parts(self, k, s):
return validate_with_humanized_errors(s, response_type.response_schema)
