def test_valid_datetime(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.SchemaField( 'a_datetime', 'A Datetime', 'datetime')) complaints = transforms.validate_object_matches_json_schema( {'a_datetime': '2014-12-17T14:10:09.222333Z'}, reg.get_json_schema_dict()) self.assertEqual(complaints, [])
def test_valid_url(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.SchemaField( 'a_url', 'A URL', 'url')) complaints = transforms.validate_object_matches_json_schema( {'a_url': 'http://x.com'}, reg.get_json_schema_dict()) self.assertEqual(complaints, [])
def test_optional_scalar_present(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.SchemaField( 'a_string', 'A String', 'string', optional=True)) complaints = transforms.validate_object_matches_json_schema( {'a_string': ''}, reg.get_json_schema_dict()) self.assertEqual(complaints, [])
def test_unexpected_member(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.SchemaField( 'a_string', 'A String', 'string')) complaints = transforms.validate_object_matches_json_schema( {'a_string': '', 'a_number': 456}, reg.get_json_schema_dict()) self.assertEqual(complaints, ['Unexpected member "a_number" in Test'])
def test_array_with_valid_content(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.FieldArray( 'scalar_array', 'Scalar Array', item_type=schema_fields.SchemaField( 'a_string', 'A String', 'string'))) complaints = transforms.validate_object_matches_json_schema( {'scalar_array': ['foo', 'bar', 'baz']}, reg.get_json_schema_dict()) self.assertEqual(complaints, [])
def test_arrays_are_implicitly_optional(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.FieldArray( 'scalar_array', 'Scalar Array', item_type=schema_fields.SchemaField( 'a_string', 'A String', 'string'))) complaints = transforms.validate_object_matches_json_schema( {}, reg.get_json_schema_dict()) self.assertEqual(complaints, [])
def test_non_struct_where_struct_expected(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.SchemaField( 'a_string', 'A String', 'string')) complaints = transforms.validate_object_matches_json_schema( 123, reg.get_json_schema_dict()) self.assertEqual( complaints, ['Expected a dict at Test, but had <type \'int\'>'])
def test_mandatory_scalar_missing(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.SchemaField( 'a_string', 'A String', 'string')) complaints = transforms.validate_object_matches_json_schema( {}, reg.get_json_schema_dict()) self.assertEqual( complaints, ['Missing mandatory value at Test.a_string'])
def test_malformed_datetime(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.SchemaField( 'a_datetime', 'A Datetime', 'datetime')) complaints = transforms.validate_object_matches_json_schema( {'a_datetime': 'not really a datetime string, is it?'}, reg.get_json_schema_dict()) self.assertEqual( complaints, ['Value "not really a datetime string, is it?" ' 'is not well-formed according to is_valid_datetime'])
def test_array_of_string(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.FieldArray( 'string_array', 'String Array', item_type=schema_fields.SchemaField(None, None, 'string'), select_data=(('one', 'One'), ('two', 'Two'), ('three', 'Three')))) json_schema = reg.get_json_schema_dict() source = {'string_array': ['one', 'two']} self.assertEqual(transforms.validate_object_matches_json_schema( source, json_schema), []) self.assertEqual(transforms.json_to_dict(source, json_schema), source)
def test_nested_dict(self): reg = schema_fields.FieldRegistry('Test') sub_registry = schema_fields.FieldRegistry('subregistry') sub_registry.add_property(schema_fields.SchemaField( 'name', 'Name', 'string', description='user name')) sub_registry.add_property(schema_fields.SchemaField( 'city', 'City', 'string', description='city name')) reg.add_sub_registry('sub_registry', title='Sub Registry', description='a sub-registry', registry=sub_registry) complaints = transforms.validate_object_matches_json_schema( {'sub_registry': {'name': 'John Smith', 'city': 'Back East'}}, reg.get_json_schema_dict()) self.assertEqual(complaints, [])
def test_array_with_bad_members(self): reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.FieldArray( 'scalar_array', 'Scalar Array', item_type=schema_fields.SchemaField( 'a_string', 'A String', 'string'))) complaints = transforms.validate_object_matches_json_schema( {'scalar_array': ['foo', 123, 'bar', 456, 'baz']}, reg.get_json_schema_dict()) self.assertEqual( complaints, ['Expected <type \'basestring\'> at Test.scalar_array[1], ' 'but instead had <type \'int\'>', 'Expected <type \'basestring\'> at Test.scalar_array[3], ' 'but instead had <type \'int\'>'])
def test_nested_dict_missing_items(self): reg = schema_fields.FieldRegistry('Test') sub_registry = schema_fields.FieldRegistry('subregistry') sub_registry.add_property(schema_fields.SchemaField( 'name', 'Name', 'string', description='user name')) sub_registry.add_property(schema_fields.SchemaField( 'city', 'City', 'string', description='city name')) reg.add_sub_registry('sub_registry', title='Sub Registry', description='a sub-registry', registry=sub_registry) complaints = transforms.validate_object_matches_json_schema( {'sub_registry': {}}, reg.get_json_schema_dict()) self.assertEqual( complaints, ['Missing mandatory value at Test.sub_registry.name', 'Missing mandatory value at Test.sub_registry.city'])
def test_array_of_dict(self): sub_registry = schema_fields.FieldRegistry('subregistry') sub_registry.add_property(schema_fields.SchemaField( 'name', 'Name', 'string', description='user name')) sub_registry.add_property(schema_fields.SchemaField( 'city', 'City', 'string', description='city name')) reg = schema_fields.FieldRegistry('Test') reg.add_property(schema_fields.FieldArray( 'struct_array', 'Struct Array', item_type=sub_registry)) complaints = transforms.validate_object_matches_json_schema( {'struct_array': [ {'name': 'One', 'city': 'Two'}, None, {'name': 'Three'}, {'city': 'Four'} ]}, reg.get_json_schema_dict()) self.assertEqual( complaints, ['Found None at Test.struct_array[1]', 'Missing mandatory value at Test.struct_array[2].city', 'Missing mandatory value at Test.struct_array[3].name'])
def reduce(user_id, values): # Convenience for collections: Pre-load Student and Course objects. student = None try: student = models.Student.get_by_user_id(user_id) # pylint: disable=broad-except except Exception: common_utils.log_exception_origin() if not student: logging.warning( 'Student for student aggregation with user ID %s ' 'was not loaded. Ignoring records for this student.', user_id) return params = context.get().mapreduce_spec.mapper.params ns = params['course_namespace'] app_context = sites.get_course_index().get_app_context_for_namespace(ns) course = courses.Course(None, app_context=app_context) # Bundle items together into lists by collection name event_items = collections.defaultdict(list) for value in values: component_name, payload = value.split(':', 1) event_items[component_name].append(transforms.loads(payload)) # Build up per-Student aggregate by calling each component. Note that # we call each component whether or not its mapper produced any # output. aggregate = {} for component in StudentAggregateComponentRegistry.get_components(): component_name = component.get_name() static_value = params.get(component_name) value = {} try: value = component.produce_aggregate( course, student, static_value, event_items.get(component_name, [])) if not value: continue # pylint: disable=broad-except except Exception, ex: common_utils.log_exception_origin() logging.critical('Student aggregation reduce function ' 'component handler %s failed: %s', component_name, str(ex)) continue schema_name = params['schema_names'][component_name] if schema_name not in value: logging.critical( 'Student aggregation reduce handler %s produced ' 'a dict which does not contain the top-level ' 'name (%s) from its registered schema.', component_name, schema_name) continue variances = transforms.validate_object_matches_json_schema( value[schema_name], params['schemas'][component_name]) if variances: logging.critical( 'Student aggregation reduce handler %s produced ' 'a value which does not match its schema: %s', component_name, ' '.join(variances)) continue aggregate.update(value)
def reduce(user_id, values): # Convenience for collections: Pre-load Student and Course objects. student = None try: student = models.Student.get_by_user_id(user_id) # pylint: disable=broad-except except Exception: common_utils.log_exception_origin() if not student: logging.warning( 'Student for student aggregation with user ID %s ' 'was not loaded. Ignoring records for this student.', user_id) return params = context.get().mapreduce_spec.mapper.params ns = params['course_namespace'] app_context = sites.get_course_index().get_app_context_for_namespace( ns) course = courses.Course(None, app_context=app_context) # Bundle items together into lists by collection name event_items = collections.defaultdict(list) for value in values: component_name, payload = value.split(':', 1) event_items[component_name].append(transforms.loads(payload)) # Build up per-Student aggregate by calling each component. Note that # we call each component whether or not its mapper produced any # output. aggregate = {} for component in StudentAggregateComponentRegistry.get_components(): component_name = component.get_name() static_value = params.get(component_name) value = {} try: value = component.produce_aggregate( course, student, static_value, event_items.get(component_name, [])) if not value: continue # pylint: disable=broad-except except Exception, ex: common_utils.log_exception_origin() logging.critical( 'Student aggregation reduce function ' 'component handler %s failed: %s', component_name, str(ex)) continue schema_name = params['schema_names'][component_name] if schema_name not in value: logging.critical( 'Student aggregation reduce handler %s produced ' 'a dict which does not contain the top-level ' 'name (%s) from its registered schema.', component_name, schema_name) continue variances = transforms.validate_object_matches_json_schema( value[schema_name], params['schemas'][component_name]) if variances: logging.critical( 'Student aggregation reduce handler %s produced ' 'a value which does not match its schema: %s', component_name, ' '.join(variances)) continue aggregate.update(value)