def _validate_ui_config(obj_type, ui_config): """Validates the value of a UI configuration.""" reference_dict = UI_CONFIG_SPECS[obj_type] assert set(ui_config.keys()) <= set(reference_dict.keys()) for key, value in ui_config.iteritems(): schema_utils.normalize_against_schema( value, reference_dict[key])
def test_normalize_against_schema_raises_exception(self): """Tests if normalize against schema raises exception for invalid key. """ with self.assertRaises(Exception): schema = {SCHEMA_KEY_TYPE: 'invalid'} schema_utils.normalize_against_schema('obj', schema)
def check_normalization(self, schema, mappings, invalid_items_with_error_messages): """Validates the schema and tests that values are normalized correctly. Args: schema: dict. The schema to normalize the value against. Each schema is a dict with at least a key called 'type'. The 'type' can take one of the SCHEMA_TYPE_* values declared above. mappings: list(tuple). A list of 2-element tuples. The first element of each item is expected to be normalized to the second. invalid_items_with_error_messages: list(tuple(str, str)). A list of values with their corresponding messages. Each value is expected to raise an AssertionError when normalized. """ validate_schema(schema) for raw_value, expected_value in mappings: self.assertEqual( schema_utils.normalize_against_schema(raw_value, schema), expected_value) for value, error_msg in invalid_items_with_error_messages: with self.assertRaisesRegexp(Exception, error_msg): schema_utils.normalize_against_schema(value, schema)
def validate_customization_arg_values(self, customization_args): """Validates customization arg values. The input is a dict whose keys are the names of the customization args. """ for ca_spec in self.customization_arg_specs: schema_utils.normalize_against_schema( customization_args[ca_spec.name]['value'], ca_spec.schema)
def validate(self): """Validates a visualization object. This is only used in tests for the validity of interactions. """ # Check that the calculation id exists. calculation_registry.Registry.get_calculation_by_id(self.calculation_id) # Check that the options_dict is valid. expected_option_names = sorted([ python_utils.convert_to_bytes( spec['name']) for spec in self._OPTIONS_SPECS]) actual_option_names = sorted(self.options.keys()) if actual_option_names != expected_option_names: raise utils.ValidationError( 'For visualization %s, expected option names %s; received ' 'names %s' % (self.id, expected_option_names, actual_option_names)) # Check that the schemas are correct. for spec in self._OPTIONS_SPECS: schema_utils.normalize_against_schema( self.options[spec['name']], spec['schema']) # Check that addressed_info_is_supported is valid. if not isinstance(self.addressed_info_is_supported, bool): raise utils.ValidationError( 'For visualization %s, expected a bool value for ' 'addressed_info_is_supported; received %s' % (self.id, self.addressed_info_is_supported))
def test_schema_key_post_normalizers(self): """Test post normalizers in schema using basic html schema.""" schema_1 = { 'type': schema_utils.SCHEMA_TYPE_HTML, 'post_normalizers': [ { 'id': 'normalize_spaces' }, # html strings with no extra spaces ] } obj_1 = 'a a' normalize_obj_1 = schema_utils.normalize_against_schema( obj_1, schema_1) self.assertEqual(u'a a', normalize_obj_1) schema_2 = { 'type': schema_utils.SCHEMA_TYPE_HTML, 'post_normalizers': [{ 'id': 'sanitize_url' }] } obj_2 = 'http://www.oppia.org/splash/<script>' normalize_obj_2 = schema_utils.normalize_against_schema( obj_2, schema_2) self.assertEqual(u'http://www.oppia.org/splash/', normalize_obj_2)
def test_normalize_against_schema_raises_exception(self): # type: () -> None """Tests if normalize against schema raises exception for invalid key. """ with self.assertRaisesRegexp(Exception, 'Invalid schema type: invalid'): # type: ignore[no-untyped-call] schema = {SCHEMA_KEY_TYPE: 'invalid'} schema_utils.normalize_against_schema('obj', schema)
def test_global_validators_raise_exception_when_error_in_list(self): with self.assertRaisesRegexp( AssertionError, r'^Validation failed: does_not_contain_email .* [email protected]$' ): obj = { 'unicodeListProp': ['*****@*****.**', 'not email 2'], 'unicodeProp': 'not email' } schema_utils.normalize_against_schema( obj, self.GLOBAL_VALIDATORS_SCHEMA, global_validators=self.GLOBAL_VALIDATORS )
def test_global_validators_raise_exception_when_error_in_dict(self): # type: () -> None with self.assertRaisesRegexp( # type: ignore[no-untyped-call] AssertionError, r'^Validation failed: does_not_contain_email .* [email protected]$' ): obj = { 'unicodeListProp': ['not email', 'not email 2'], 'unicodeProp': '*****@*****.**' } schema_utils.normalize_against_schema( obj, self.GLOBAL_VALIDATORS_SCHEMA, global_validators=self.GLOBAL_VALIDATORS)
def normalize(cls, raw): """Validates and normalizes a raw Python object.""" """ Checks that there are no self-loops or multiple edges. Checks that unlabeled graphs have all labels empty. Checks that unweighted graphs have all weights set to 1. TODO(czx): Think about support for multigraphs? """ try: raw = schema_utils.normalize_against_schema(raw, cls.SCHEMA) if not raw["isLabeled"]: for vertex in raw["vertices"]: assert vertex["label"] == "" for edge in raw["edges"]: assert edge["src"] != edge["dst"] if not raw["isWeighted"]: assert edge["weight"] == 1.0 if raw["isDirected"]: edge_pairs = [(edge["src"], edge["dst"]) for edge in raw["edges"]] else: edge_pairs = [(edge["src"], edge["dst"]) for edge in raw["edges"]] + [ (edge["dst"], edge["src"]) for edge in raw["edges"] ] assert len(set(edge_pairs)) == len(edge_pairs) except Exception: raise TypeError("Cannot convert to graph %s" % raw) return raw
def _validate_customization_arg_specs(self, customization_args): """Validates the customization arg specs for the interaction. Args: customization_args: list(dict(str, *)). The customization args for the interaction. """ for ca_spec in customization_args: self.assertEqual(set(ca_spec.keys()), set([ 'name', 'description', 'schema', 'default_value'])) self.assertTrue( isinstance(ca_spec['name'], python_utils.BASESTRING)) self.assertTrue(self._is_alphanumeric_string(ca_spec['name'])) self.assertTrue( isinstance(ca_spec['description'], python_utils.BASESTRING)) self.assertGreater(len(ca_spec['description']), 0) schema_utils_test.validate_schema(ca_spec['schema']) self.assertEqual( ca_spec['default_value'], schema_utils.normalize_against_schema( ca_spec['default_value'], ca_spec['schema'])) if ca_spec['schema']['type'] == 'custom': obj_class = obj_services.Registry.get_object_class_by_type( ca_spec['schema']['obj_type']) self.assertEqual( ca_spec['default_value'], obj_class.normalize(ca_spec['default_value']))
def normalize(cls, raw): """Validates and normalizes a raw Python object.""" """ Checks that there are no self-loops or multiple edges. Checks that unlabeled graphs have all labels empty. Checks that unweighted graphs have all weights set to 1. TODO(czx): Think about support for multigraphs? """ try: raw = schema_utils.normalize_against_schema(raw, cls.SCHEMA) if not raw['isLabeled']: for vertex in raw['vertices']: assert (vertex['label'] == '') for edge in raw['edges']: assert (edge['src'] != edge['dst']) if not raw['isWeighted']: assert (edge['weight'] == 1.0) if raw['isDirected']: edge_pairs = [ (edge['src'], edge['dst']) for edge in raw['edges']] else: edge_pairs = ( [(edge['src'], edge['dst']) for edge in raw['edges']] + [(edge['dst'], edge['src']) for edge in raw['edges']] ) assert len(set(edge_pairs)) == len(edge_pairs) except Exception: raise TypeError('Cannot convert to graph %s' % raw) return raw
def _validate_customization_arg_specs(self, customization_args): """Validates the customization arg specs for the interaction. Args: customization_args: list(CustomizationArgSpec). The customization args for the interaction. """ for ca_spec in customization_args: self.assertTrue( all( hasattr(ca_spec, attr) for attr in ['name', 'description', 'schema', 'default_value'])) self.assertTrue(isinstance(ca_spec.name, python_utils.BASESTRING)) self.assertTrue(self._is_alphanumeric_string(ca_spec.name)) self.assertTrue( isinstance(ca_spec.description, python_utils.BASESTRING)) self.assertGreater(len(ca_spec.description), 0) schema_utils_test.validate_schema(ca_spec.schema) self.assertEqual( ca_spec.default_value, schema_utils.normalize_against_schema(ca_spec.default_value, ca_spec.schema)) if ca_spec.schema['type'] == 'custom': obj_class = object_registry.Registry.get_object_class_by_type( ca_spec.schema['obj_type']) self.assertEqual(ca_spec.default_value, obj_class.normalize(ca_spec.default_value))
def _validate_customization_arg_specs(self, customization_arg_specs): for ca_spec in customization_arg_specs: self.assertEqual(set(ca_spec.keys()), set([ 'name', 'description', 'schema', 'default_value'])) self.assertTrue(isinstance(ca_spec['name'], basestring)) self.assertTrue(self._is_alphanumeric_string(ca_spec['name'])) self.assertTrue(isinstance(ca_spec['description'], basestring)) self.assertGreater(len(ca_spec['description']), 0) # The default value might not pass validation checks (e.g. the # Image component has a required field whose default value is # empty). Thus, when checking the default value schema, we don't # apply the custom validators. schema_utils_test.validate_schema(ca_spec['schema']) self.assertEqual( ca_spec['default_value'], schema_utils.normalize_against_schema( ca_spec['default_value'], ca_spec['schema'], apply_custom_validators=False)) if ca_spec['schema']['type'] == 'custom': obj_class = obj_services.Registry.get_object_class_by_type( ca_spec['schema']['obj_type']) self.assertIsNotNone(obj_class.edit_html_filename) self.assertIsNotNone(obj_class.edit_js_filename) self.assertEqual( ca_spec['default_value'], obj_class.normalize(ca_spec['default_value']))
def normalize(cls, raw): # Moves cur_value to the nearest available value in the range # [min_value, max_value]. """Returns the normalized coordinates of the rectangle. Args: raw: *. An object to be validated against the schema, normalizing if necessary. Returns: list(list(float)). The normalized object containing list of lists of float values as coordinates of the rectangle. Raises: TypeError: Cannot convert to the NormalizedRectangle2D schema. """ def clamp(min_value, current_value, max_value): return min(max_value, max(min_value, current_value)) try: raw = schema_utils.normalize_against_schema(raw, cls.SCHEMA) raw[0][0] = clamp(0.0, raw[0][0], 1.0) raw[0][1] = clamp(0.0, raw[0][1], 1.0) raw[1][0] = clamp(0.0, raw[1][0], 1.0) raw[1][1] = clamp(0.0, raw[1][1], 1.0) except Exception: raise TypeError('Cannot convert to Normalized Rectangle %s' % raw) return raw
def _validate_customization_arg_specs(self, customization_arg_specs): for ca_spec in customization_arg_specs: self.assertEqual( set(ca_spec.keys()), set(['name', 'description', 'schema', 'default_value'])) self.assertTrue(isinstance(ca_spec['name'], basestring)) self.assertTrue(self._is_alphanumeric_string(ca_spec['name'])) self.assertTrue(isinstance(ca_spec['description'], basestring)) self.assertGreater(len(ca_spec['description']), 0) # The default value might not pass validation checks (e.g. the # Image component has a required field whose default value is # empty). Thus, when checking the default value schema, we don't # apply the custom validators. schema_utils_test.validate_schema(ca_spec['schema']) self.assertEqual( ca_spec['default_value'], schema_utils.normalize_against_schema( ca_spec['default_value'], ca_spec['schema'], apply_custom_validators=False)) if ca_spec['schema']['type'] == 'custom': obj_class = obj_services.Registry.get_object_class_by_type( ca_spec['schema']['obj_type']) self.assertEqual(ca_spec['default_value'], obj_class.normalize(ca_spec['default_value']))
def _validate_validator(obj_type, validator): # type: (Text, Dict[Text, Any]) -> None """Validates the value of a 'validator' field. Args: obj_type: str. The type of the object. validator: dict. The Specs that needs to be validated. Raises: AssertionError. The object fails to validate against the schema. """ reference_dict = VALIDATOR_SPECS[obj_type] assert 'id' in validator, 'id is not present in validator' assert validator['id'] in reference_dict, ( '%s is not present in reference_dict' % validator['id']) customization_keys = list(validator.keys()) customization_keys.remove('id') assert ( set(customization_keys) == set(reference_dict[validator['id']].keys())), ( 'Missing keys: %s, Extra keys: %s' % ( list( set(reference_dict[validator['id']].keys()) - set(customization_keys)), list( set(customization_keys) - set(reference_dict[validator['id']].keys())))) for key in customization_keys: value = validator[key] schema = reference_dict[validator['id']][key] try: schema_utils.normalize_against_schema(value, schema) except Exception as e: raise AssertionError(e) # Check that the id corresponds to a valid normalizer function. validator_fn = schema_utils.get_validator(validator['id']) assert set(inspect.getargspec(validator_fn).args) == set( customization_keys + ['obj']), ( 'Missing keys: %s, Extra keys: %s' % ( list( set(customization_keys + ['obj']) - set(inspect.getargspec(validator_fn).args)), list( set(inspect.getargspec(validator_fn).args) - set(customization_keys + ['obj']))))
def validate( handler_args: Any, handler_args_schemas: Any, allowed_extra_args: bool, allow_string_to_bool_conversion: bool = False ) -> Tuple[Dict[str, Any], List[str]]: """Calls schema utils for normalization of object against its schema and collects all the errors. Args: handler_args: *. Object for normalization. handler_args_schemas: dict. Schema for args. allowed_extra_args: bool. Whether extra args are allowed in handler. allow_string_to_bool_conversion: bool. Whether to allow string to boolean coversion. Returns: *. A two tuple, where the first element represents the normalized value in dict format and the second element represents the lists of errors after validation. """ # Collect all errors and present them at once. errors = [] normalized_value = {} for arg_key, arg_schema in handler_args_schemas.items(): if arg_key not in handler_args or handler_args[arg_key] is None: if ('default_value' in arg_schema and arg_schema['default_value'] is None): # Skip validation for optional cases. continue elif ('default_value' in arg_schema and arg_schema['default_value'] is not None): handler_args[arg_key] = arg_schema['default_value'] elif 'default_value' not in arg_schema: errors.append('Missing key in handler args: %s.' % arg_key) continue # Below normalization is for arguments which are expected to be boolean # but from API request they are received as string type. if (allow_string_to_bool_conversion and arg_schema['schema']['type'] == schema_utils.SCHEMA_TYPE_BOOL and isinstance( handler_args[arg_key], python_utils.BASESTRING)): handler_args[arg_key] = (convert_string_to_bool( handler_args[arg_key])) try: normalized_value[arg_key] = schema_utils.normalize_against_schema( handler_args[arg_key], arg_schema['schema']) except Exception as e: errors.append('Schema validation for \'%s\' failed: %s' % (arg_key, e)) extra_args = set(handler_args.keys()) - set(handler_args_schemas.keys()) if not allowed_extra_args and extra_args: errors.append('Found extra args: %s.' % (list(extra_args))) return normalized_value, errors
def _validate_ui_config(obj_type, ui_config): """Validates the value of a UI configuration. Args: obj_type: str. UI config spec type. ui_config: dict. The UI config that needs to be validated. Raises: AssertionError. The object fails to validate against the schema. """ reference_dict = UI_CONFIG_SPECS[obj_type] assert set(ui_config.keys()) <= set(reference_dict.keys()), ( 'Missing keys: %s, Extra keys: %s' % ( list(set(reference_dict.keys()) - set(ui_config.keys())), list(set(ui_config.keys()) - set(reference_dict.keys())))) for key, value in ui_config.items(): schema_utils.normalize_against_schema( value, reference_dict[key])
def check_normalization(self, schema, mappings, invalid_items): """Validates the schema and tests that values are normalized correctly. Args: schema: the schema to normalize the value against. mappings: a list of 2-element tuples. The first element of each item is expected to be normalized to the second. invalid_items: a list of values. Each of these is expected to raise an AssertionError when normalized. """ validate_schema(schema) for raw_value, expected_value in mappings: self.assertEqual( schema_utils.normalize_against_schema(raw_value, schema), expected_value) for value in invalid_items: with self.assertRaises(Exception): schema_utils.normalize_against_schema(value, schema)
def test_global_validators_pass_when_no_error(self): obj = { 'unicodeListProp': ['not email', 'not email 2'], 'unicodeProp': 'not email' } normalized_obj = schema_utils.normalize_against_schema( obj, self.GLOBAL_VALIDATORS_SCHEMA, global_validators=self.GLOBAL_VALIDATORS ) self.assertEqual(obj, normalized_obj)
def normalize(cls, raw): """Validates and normalizes a raw Python object. Returns: a normalized Python object describing the Object specified by this class. Raises: TypeError: if the Python object cannot be normalized. """ return schema_utils.normalize_against_schema(raw, cls.SCHEMA)
def normalize(self, value): """Validates the given object using the schema and normalizes if necessary. Args: value: The value of the configuration property. Returns: instance. The normalized object. """ return schema_utils.normalize_against_schema(value, self._schema)
def __init__(self, name, schema, description, default_value): if Registry.get_config_property(name): raise Exception('Property with name %s already exists' % name) self._name = name self._schema = schema self._description = description self._default_value = schema_utils.normalize_against_schema( default_value, self._schema) Registry.init_config_property(self.name, self)
def __init__(self, name, schema, description, default_value): if name in Registry._config_registry: raise Exception('Property with name %s already exists' % name) self._name = name # TODO(sll): Validate the schema. self._schema = schema self._description = description self._default_value = schema_utils.normalize_against_schema( default_value, self._schema) Registry._config_registry[self.name] = self
def _validate_validator(obj_type, validator): """Validates the value of a 'validator' field.""" reference_dict = VALIDATOR_SPECS[obj_type] assert 'id' in validator and validator['id'] in reference_dict customization_keys = validator.keys() customization_keys.remove('id') assert (set(customization_keys) == set(reference_dict[validator['id']].keys())) for key in customization_keys: value = validator[key] schema = reference_dict[validator['id']][key] try: schema_utils.normalize_against_schema(value, schema) except Exception as e: raise AssertionError(e) # Check that the id corresponds to a valid normalizer function. validator_fn = schema_utils._Validators.get(validator['id']) assert set(inspect.getargspec(validator_fn).args) == set( customization_keys + ['obj'])
def validate(self): """Validates a visualization object. This is only used in tests for the validity of interactions. """ # Check that the calculation id exists. calculation_registry.Registry.get_calculation_by_id(self.calculation_id) # Check that the options_dict is valid. expected_option_names = sorted([ spec['name'] for spec in self._OPTIONS_SPECS]) actual_option_names = sorted(self.options.keys()) if actual_option_names != expected_option_names: raise utils.ValidationError( 'For visualization %s, expected option names %s; received ' 'names %s' % (self.id, expected_option_names, actual_option_names)) # Check that the schemas are correct. for spec in self._OPTIONS_SPECS: schema_utils.normalize_against_schema( self.options[spec['name']], spec['schema'])
def normalize(self, value): """Validates the given object using the schema and normalizes if necessary. Args: value: str. The value of the configuration property. Returns: instance. The normalized object. """ email_validators = [{'id': 'does_not_contain_email'}] return schema_utils.normalize_against_schema( value, self._schema, global_validators=email_validators)
def normalize(cls, raw): """Validates and normalizes a raw Python object. Args: raw: *. A normalized Python object to be normalized. Returns: *. A normalized Python object describing the Object specified by this class. Raises: TypeError. The Python object cannot be normalized. """ return schema_utils.normalize_against_schema(raw, cls.get_schema())
def validate(self): """Validates a visualization object. This is only used in tests for the validity of interactions. """ # Check that the calculation id exists. calculation_registry.Registry.get_calculation_by_id( self.calculation_id) # Check that the options_dict is valid. expected_option_names = sorted( [spec['name'] for spec in self._OPTIONS_SPECS]) actual_option_names = sorted(self.options.keys()) if actual_option_names != expected_option_names: raise utils.ValidationError( 'For visualization %s, expected option names %s; received ' 'names %s' % (self.id, expected_option_names, actual_option_names)) # Check that the schemas are correct. for spec in self._OPTIONS_SPECS: schema_utils.normalize_against_schema(self.options[spec['name']], spec['schema'])
def __init__(self, name, schema, description, default_value, post_set_hook=None, is_directly_settable=True): if Registry.get_config_property(name): raise Exception('Property with name %s already exists' % name) self._name = name self._schema = schema self._description = description self._default_value = schema_utils.normalize_against_schema( default_value, self._schema) self._post_set_hook = post_set_hook self._is_directly_settable = is_directly_settable Registry.init_config_property(self.name, self)
def normalize(cls, raw): """Validates and normalizes a raw Python object. Args: raw: *. A Python object to be validated against the schema, normalizing if necessary. Returns: bool. The normalized object (or False if the input is None or ''). """ if raw is None or raw == '': raw = False return schema_utils.normalize_against_schema(raw, cls.SCHEMA)
def normalize(cls, raw): """Validates and normalizes a raw Python object. Args: raw: *. A Python object to be validated against the schema, normalizing if necessary. Returns: unicode. The normalized object containing string in unicode format. """ if '\t' in raw: raise TypeError('Unexpected tab characters in code string: %s' % raw) return schema_utils.normalize_against_schema(raw, cls.SCHEMA)
def normalize(cls, raw): # Moves cur_value to the nearest available value in the range [min_value, max_value] def clamp(min_value, current_value, max_value): return min(max_value, max(min_value, current_value)) try: raw = schema_utils.normalize_against_schema(raw, cls.SCHEMA) raw[0][0] = clamp(0.0, raw[0][0], 1.0) raw[0][1] = clamp(0.0, raw[0][1], 1.0) raw[1][0] = clamp(0.0, raw[1][0], 1.0) raw[1][1] = clamp(0.0, raw[1][1], 1.0) except Exception: raise TypeError('Cannot convert to Normalized Rectangle %s' % raw) return raw
def _validate_customization_arg_specs(self, customization_args): for ca_spec in customization_args: self.assertEqual(set(ca_spec.keys()), set([ 'name', 'description', 'schema', 'default_value'])) self.assertTrue(isinstance(ca_spec['name'], basestring)) self.assertTrue(self._is_alphanumeric_string(ca_spec['name'])) self.assertTrue(isinstance(ca_spec['description'], basestring)) self.assertGreater(len(ca_spec['description']), 0) schema_utils_test.validate_schema(ca_spec['schema']) self.assertEqual( ca_spec['default_value'], schema_utils.normalize_against_schema( ca_spec['default_value'], ca_spec['schema'])) if ca_spec['schema']['type'] == 'custom': obj_class = obj_services.Registry.get_object_class_by_type( ca_spec['schema']['obj_type']) self.assertIsNotNone(obj_class.edit_html_filename) self.assertIsNotNone(obj_class.edit_js_filename) self.assertEqual( ca_spec['default_value'], obj_class.normalize(ca_spec['default_value']))
def normalize(cls, raw): """Reads `raw` as a unicode string representing a tar file and returns the base64-encoded contents.""" raw = schema_utils.normalize_against_schema(raw, cls._schema) raw = base64.b64decode(raw) return tarfile.open(fileobj=StringIO.StringIO(raw), mode='r:gz')
def normalize(self, value): return schema_utils.normalize_against_schema(value, self._schema)
def normalize(cls, raw): if '\t' in raw: raise TypeError( 'Unexpected tab characters in code string: %s' % raw) return schema_utils.normalize_against_schema(raw, cls.SCHEMA)
def normalize(cls, raw): """Validates and normalizes a raw Python object.""" if raw is None or raw == '': raw = False return schema_utils.normalize_against_schema(raw, cls.SCHEMA)