def validate(self): """validates the JSON Raises: InvalidServerSettings: bad entry in the JSON """ try: humanize.validate_with_humanized_errors(self.settings, self.schema) except voluptuous.Error as error: self.log.error(error) raise InvalidSettings("bad settings {}".format(error)) return
def validateItems(**kwargs): for k in kwargs.keys(): if k not in __validator__: print("ERROR! got unrecognized key: {}".format(k)) return 0 S = Schema({k: v for k, v in __validator__.items()}, extra=PREVENT_EXTRA) return validate_with_humanized_errors(kwargs, S)
def assemble_args(self, **options): options = validate_with_humanized_errors(options, cudaDeconSchema) arglist = [] for o in options: # convert LLSpy variable naming conventions to cudaDeconv names # TODO: consider uniying everything to cudaDeconv? optname = o convert_name = { 'nIters': 'RL', } if optname in convert_name: optname = convert_name[optname] # assemble the argument list if self.has_option_longname(optname): # expand listed items like --MIP 0 0 0 if optname in ('MIP', 'rMIP', 'crop'): arglist.append('--' + optname) [arglist.append(str(i)) for i in options[o]] # booleans only get a single flag elif isinstance(options[o], bool): if options[o]: arglist.extend(['--' + optname]) # otherwise just add the argument else: arglist.extend(['--' + optname, str(options[o])]) else: logger.warn( 'Warning: option not recognized, ignoring: {}'.format(o)) return arglist
def post(self): try: vhum.validate_with_humanized_errors(request.json, self.LOGIN_VALIDATOR) except verr.Error as invalid: raise BadRequest(str(invalid)) try: user = UserAccount.query.filter_by( email=request.json.get('email')).one() except NoResultFound: raise Unauthorized('Email or password incorrect') if not user.verify_password(request.json.get('password')): raise Unauthorized('Email or password incorrect') token = user.generate_auth_token() return {'access_token': token.decode('ascii')}
def __init__(self, *args, **kwargs): """ Service configuration class constructor """ super(ServiceConfiguration, self).__init__() self.dictionary = validate_with_humanized_errors( kwargs, self.VALIDATION_SCHEMA)
def load(cls, src): articles = yaml.safe_load(src) # validate validated = validate_with_humanized_errors(articles, ARTICLES) # instantiate rules = [Article.from_db(a) for a in validated] return cls(rules)
def validate_parts(self, k, parts): ## add for submeasure by datang 11/9/2019 ##if (len(parts)==1 and parts[0].keys[0]=='submeasure'): ## return parts parts = validate_with_humanized_errors( parts, response_type.response_parts_schema) response_type.validate_parts(parts) return parts
def get_mozilla_office_ips(): try: mozilla_ip_ranges_file = Path('/var/hg/moz-ip-ranges.txt') bloxtool_config_file = Path('/etc/mercurial/bloxtool.ini') bloxtool_command = [ '/var/hg/venv_tools/bin/bloxtool', 'network', 'search', 'attribute', 'subnet-purpose', 'value', 'nat-pool', '--format=json', f'--config={bloxtool_config_file}' ] bloxtool_json_schema = Schema([{ 'comment': str, '_ref': str, 'network': is_ip_address_network, 'network_view': str, }], extra=False, required=True) # Get raw string output and convert to Python dict process_output = subprocess.run(bloxtool_command, check=True, encoding='utf-8', stdout=subprocess.PIPE).stdout output_as_dict = json.loads(process_output) # Verify dict schema validate_with_humanized_errors(output_as_dict, bloxtool_json_schema) write_to_file_atomically( mozilla_ip_ranges_file, '\n'.join(i['network'] for i in output_as_dict)) except subprocess.CalledProcessError as cpe: sys.exit('An error occurred while executing the bloxtool command.') except json.JSONDecodeError as jde: sys.exit('An error occurred parsing the bloxtool output as JSON.') except VoluptuousInvalid as vi: sys.exit( 'The JSON data from bloxtool does not match the required schema.')
def post(self): try: vhum.validate_with_humanized_errors(request.json, self.REGISTER_VALIDATOR) except verr.Error as invalid: raise BadRequest(str(invalid)) email = request.json['email'] user_with_same_email = UserAccount.query.filter( UserAccount.email == email).one_or_none() if user_with_same_email: raise Conflict(f"Email '{email}' is already taken.") password = request.json.pop('password') new_user = UserAccount(**request.json) new_user.hash_password(password) db.session.add(new_user) db.session.commit() return new_user, 201
def on_get(self, req, resp, word): """ Return http response Get method for get tweets of word """ data = req.params validate_with_humanized_errors(data, SearchTweets.validator) if 'limit' in data and data.get('limit', '').isdigit(): limit = int(data.get('limit')) elif 'limit' in data and not data.get('limit', '').isdigit(): raise falcon.HTTPBadRequest('Invalid parameter in endpoint') else: limit = self.default try: tweet_word = TwitterSearch(word) tweets = tweet_word.get_tweets(limit) resp.body = tweets resp.status = falcon.HTTP_200 except Exception as error: resp.status = falcon.HTTP_404
def localParams(*args, **kwargs): """ returns a validated dict of processing parameters with defaults filled in when not supplied, that ALSO contains parameters to a specific LLSdir instance. returned by llspy.llsdir.localParams() """ if len(args) == 1 and isinstance(args[0], dict): kwargs = args[0] S = validate_with_humanized_errors(kwargs, __localSchema__) return dotdict(S)
def on_get(self, req, resp, user): """ Return http response Get method for get tweets of user """ data = req.params validate_with_humanized_errors(data, UserTweets.validator) if 'limit' in data and data.get('limit', '').isdigit(): limit = int(data.get('limit')) elif 'limit' in data and not data.get('limit', '').isdigit(): raise falcon.HTTPBadRequest('Invalid parameter in endpoint') else: limit = self.default try: tweet_user = TwitterUser(user) tweets = tweet_user.get_tweets(limit) resp.body = tweets resp.status = falcon.HTTP_200 except Exception as error: resp.body = json.dumps({'Error': str(error)}) resp.status = falcon.HTTP_404
def procParams(*args, **kwargs): """ returns a validated dict of processing parameters with defaults filled in when not supplied. >>> P = procParams() # get default parameters >>> P = procParams(nIters=7, tRange=range(0,10)) # check validitity of parameter name >>> 'regMode' in procParams() """ # accept a single dict as well as expanded options if len(args) == 1 and isinstance(args[0], dict): kwargs = args[0] S = validate_with_humanized_errors(kwargs, __schema__) if S['nIters'] > 0 and S['otfDir'] is None: raise ParametersError('oftDir cannot be type None with nIters > 0') return dotdict(S)
def validate(schema, data): # TODO - print valid keys when not allowed key is used try: return validate_with_humanized_errors(data, Schema(schema)) except Exception as e: raise e
def get_mozilla_office_ips(): '''Entry point for the Mozilla office IP scraper Calls out to bloxtool to obtain Mozilla network information in JSON format. Validates the JSON against a known schema and atomically re-writes a file with the CIDR representations of Mozilla office IP address spaces. ''' try: mozilla_ip_ranges_file = Path('/var/hg/moz-ip-ranges.txt') bloxtool_config_file = Path('/etc/mercurial/bloxtool.ini') bloxtool_command = [ '/var/hg/venv_tools/bin/bloxtool', 'network', 'search', 'attribute', 'subnet-purpose', 'value', 'nat-pool', '--format=json', f'--config={bloxtool_config_file}' ] bloxtool_json_schema = Schema([{ 'comment': str, '_ref': str, 'network': is_ip_address_network, 'network_view': str, }], extra=False, required=True) # Get raw string output and convert to Python dict process_output = subprocess.run(bloxtool_command, check=True, encoding='utf-8', stdout=subprocess.PIPE).stdout output_as_dict = json.loads(process_output) # Verify dict schema validate_with_humanized_errors(output_as_dict, bloxtool_json_schema) write_to_file_atomically( mozilla_ip_ranges_file, '\n'.join(i['network'] for i in output_as_dict)) except subprocess.CalledProcessError as cpe: logger.exception( 'An error occurred while executing the bloxtool command: exit code %s' % cpe.returncode) logger.exception('STDOUT: %s' % cpe.stdout) logger.exception('STDERR: %s' % cpe.stderr) sys.exit(1) except json.JSONDecodeError as jde: logger.exception( 'An error occurred parsing the bloxtool output as JSON: %s' % jde.msg) sys.exit(1) except VoluptuousInvalid as vi: logger.exception( 'The JSON data from bloxtool does not match the required schema.') logger.exception('Error message: %s' % vi.msg) logger.exception('Error path: %s' % vi.path) logger.exception('Exception message: %s' % vi.error_message) sys.exit(1)
def get_aws_ips(): '''Entry point for the AWS IP address scraper Downloads the AWS IP ranges JSON document from Amazon and verifies against a known schema. Atomically rewrites a file with the CIDR representations of AWS IP address spaces. ''' try: # Grab the new data from Amazon amazon_ip_ranges_file = Path('/var/hg/aws-ip-ranges.json') ip_ranges_response = requests.get( 'https://ip-ranges.amazonaws.com/ip-ranges.json') # Ensure 200 OK response code if ip_ranges_response.status_code != 200: sys.exit('HTTP response from Amazon was not 200 OK') # Sanity check: ensure the file is an appropriate size if len(ip_ranges_response.content) < 88000: sys.exit( 'The retrieved AWS JSON document is smaller than the minimum allowable file size' ) # JSON Schema for the Amazon IP Ranges JSON document amazon_json_schema = Schema( { 'syncToken': str, 'createDate': str, 'ipv6_prefixes': [ dict ], # If IPv6 is supported in the future, this will need to be defined # The prefixes field must meet both requirements: # 1. There must be at least one entry for each region containing CI and S3 bundles # 2. Must be a list of dicts that fit the schema below 'prefixes': All(all_required_regions_exist, [ { 'ip_prefix': is_ip_address_network, 'region': str, 'service': str, }, ]), }, extra=False, required=True) # Validate dict schema output_as_dict = ip_ranges_response.json() validate_with_humanized_errors(output_as_dict, amazon_json_schema) # Sanity check: ensure the syncToken indicates an IP space change has been made # since the last recorded change. Only check if a file exists, in case of new deployments if amazon_ip_ranges_file.is_file(): file_bytes = amazon_ip_ranges_file.read_bytes() existing_document_as_dict = json.loads(file_bytes) file_diff = diff(existing_document_as_dict, output_as_dict, context=0) # Exit if the file contents are the same or the syncToken has not changed if not file_diff or int(output_as_dict['syncToken']) <= int( existing_document_as_dict['syncToken']): sys.exit() else: existing_document_as_dict = { } # No existing document means whole file is the diff file_diff = diff(existing_document_as_dict, output_as_dict, context=0) write_to_file_atomically(amazon_ip_ranges_file, json.dumps(output_as_dict)) # Print the diff for collection as systemd unit output logger.info('AWS IP ranges document has been updated') logger.info(file_diff) except subprocess.CalledProcessError as cpe: logger.exception( 'An error occurred when notifying about changes to the file: exit code %s' % cpe.returncode) logger.exception('STDOUT: %s' % cpe.stdout) logger.exception('STDERR: %s' % cpe.stderr) sys.exit(1) except json.JSONDecodeError as jde: logger.exception( 'An error occurred parsing the data retrieved from Amazon as JSON: %s' % jde.msg) sys.exit(1) except VoluptuousInvalid as vi: logger.exception( 'The JSON data from Amazon does not match the required schema.') logger.exception('Error message: %s' % vi.msg) logger.exception('Error path: %s' % vi.path) logger.exception('Exception message: %s' % vi.error_message) sys.exit(1)
def validate_structure(self, k, s): return validate_with_humanized_errors(s, Survey._structure_schema)
def verify(data, *validators): assert validate_with_humanized_errors(data, Schema(All(*validators))) == data
ATTRIBUTE_SCHEMA = All(Schema({str: [BROADCAST_SCHEMA]}), validate_broadcast) MODULE_INFO_SCHEMA = All( Schema({ Required('code'): Coerce(int), Required('protocol'): All(Lower, Any('any', Coerce(int))), Required('software_versions'): All([Coerce(int)]), Required('description'): Coerce(str), Required('attributes'): ATTRIBUTE_SCHEMA, })) SCHEMA = All(Schema({ str: MODULE_INFO_SCHEMA, })) validate_with_humanized_errors(data=module_data, schema=SCHEMA) module_info = SCHEMA(module_data) class ModuleFactory: """This is a ModuleFactory class to create specific module class based on discovered data.""" factories = {} @staticmethod def create_module(can_id, code, protocol, software_version): """Create module object based on proivded data. Args: can_id (int): CAN ID code (int): Module Code Number
def verify(data, *validators): assert validate_with_humanized_errors(data, Schema(All(*validators))) == data
'network_view': str, }], extra=False, required=True) if __name__ == '__main__': try: # Get raw string output and convert to Python dict process_output = subprocess.run(COMMAND, check=True, encoding='utf-8', stdout=subprocess.PIPE).stdout output_as_dict = json.loads(process_output) # Verify dict schema validate_with_humanized_errors(output_as_dict, JSON_SCHEMA) # Write data to a temp file, atomically rewrite the the IP ranges file temp_file_path = IP_RANGES_FILE.with_suffix('.tmp') with temp_file_path.open(mode='w') as temp_file: temp_file.write('\n'.join(i['network'] for i in output_as_dict)) temp_file_path.rename(IP_RANGES_FILE) except subprocess.CalledProcessError as cpe: sys.exit('An error occurred while executing the bloxtool command.') except json.JSONDecodeError as jde: sys.exit('An error occurred parsing the bloxtool output as JSON.') except VoluptuousInvalid as vi:
def validate_dict(cls, dct): return validate_with_humanized_errors(dct, cls.schema)
def validate_settings(self, premodel_settings, session_type): ''' This method validates the premodel settings for the 'data_new', 'data_append', 'model_generate', or 'model_predict' sessions. Note: This method does not validate the associated 'file upload(s)', which is the responsibility of the mongodb query process. ''' # local variables model_type = current_app.config.get('MODEL_TYPE') dataset_type = current_app.config.get('DATASET_TYPE') sv_kernel_type = current_app.config.get('SV_KERNEL_TYPE') # validation on 'data_new', 'data_append' session if session_type in ['data_new', 'data_append']: if premodel_settings[ 'stream'] == 'True' or session_type == 'data_new': schema = Schema({ Required('collection'): All(unicode, Length(min=1)), Required('dataset_type'): In(dataset_type), Required('model_type'): In(model_type), Required('session_type'): Any('data_new', 'data_append'), Required('session_name'): All(unicode, Length(min=1)), Optional('stream'): Any('True', 'False'), }) else: schema = Schema({ Required('collection'): All(unicode, Length(min=1)), Required('dataset_type'): In(dataset_type), Required('model_type'): In(model_type), Required('session_type'): Any('data_new', 'data_append'), Optional('stream'): Any('True', 'False'), }) # validation on 'model_generate' session if session_type == 'model_generate': schema = Schema({ Required('collection'): All(unicode, Length(min=1)), Required('model_type'): In(model_type), Required('session_type'): 'model_generate', Optional('stream'): Any('True', 'False'), Required('sv_kernel_type'): In(sv_kernel_type), Optional('gamma'): Any(Coerce(int), Coerce(float)), Optional('penalty'): Any(Coerce(int), Coerce(float)), }) # validation on 'model_predict' session elif session_type == 'model_predict': schema = Schema({ Required('collection'): All(unicode, Length(min=1)), Optional('stream'): Any('True', 'False'), Required('prediction_input[]'): [ Any(Coerce(int), Coerce(float)), ], Required('session_type'): 'model_predict', }) try: validate_with_humanized_errors(premodel_settings, schema) except Exception, error: split_error = str(error).splitlines() self.list_error.append(split_error) return split_error
def validate_response_parts(self, k, s): return validate_with_humanized_errors(s, response_type.response_schema)