def test_valid_datetime(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.SchemaField(
'a_datetime', 'A Datetime', 'datetime'))
complaints = transforms.validate_object_matches_json_schema(
{'a_datetime': '2014-12-17T14:10:09.222333Z'},
reg.get_json_schema_dict())
self.assertEqual(complaints, [])
def test_valid_url(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.SchemaField(
'a_url', 'A URL', 'url'))
complaints = transforms.validate_object_matches_json_schema(
{'a_url': 'http://x.com'},
reg.get_json_schema_dict())
self.assertEqual(complaints, [])
def test_optional_scalar_present(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.SchemaField(
'a_string', 'A String', 'string', optional=True))
complaints = transforms.validate_object_matches_json_schema(
{'a_string': ''},
reg.get_json_schema_dict())
self.assertEqual(complaints, [])
def test_unexpected_member(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.SchemaField(
'a_string', 'A String', 'string'))
complaints = transforms.validate_object_matches_json_schema(
{'a_string': '',
'a_number': 456},
reg.get_json_schema_dict())
self.assertEqual(complaints, ['Unexpected member "a_number" in Test'])
def test_array_with_valid_content(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.FieldArray(
'scalar_array', 'Scalar Array',
item_type=schema_fields.SchemaField(
'a_string', 'A String', 'string')))
complaints = transforms.validate_object_matches_json_schema(
{'scalar_array': ['foo', 'bar', 'baz']},
reg.get_json_schema_dict())
self.assertEqual(complaints, [])
def test_arrays_are_implicitly_optional(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.FieldArray(
'scalar_array', 'Scalar Array',
item_type=schema_fields.SchemaField(
'a_string', 'A String', 'string')))
complaints = transforms.validate_object_matches_json_schema(
{},
reg.get_json_schema_dict())
self.assertEqual(complaints, [])
def test_non_struct_where_struct_expected(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.SchemaField(
'a_string', 'A String', 'string'))
complaints = transforms.validate_object_matches_json_schema(
123,
reg.get_json_schema_dict())
self.assertEqual(
complaints,
['Expected a dict at Test, but had <type \'int\'>'])
def test_mandatory_scalar_missing(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.SchemaField(
'a_string', 'A String', 'string'))
complaints = transforms.validate_object_matches_json_schema(
{},
reg.get_json_schema_dict())
self.assertEqual(
complaints,
['Missing mandatory value at Test.a_string'])
def test_malformed_datetime(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.SchemaField(
'a_datetime', 'A Datetime', 'datetime'))
complaints = transforms.validate_object_matches_json_schema(
{'a_datetime': 'not really a datetime string, is it?'},
reg.get_json_schema_dict())
self.assertEqual(
complaints,
['Value "not really a datetime string, is it?" '
'is not well-formed according to is_valid_datetime'])
def test_array_of_string(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.FieldArray(
'string_array', 'String Array',
item_type=schema_fields.SchemaField(None, None, 'string'),
select_data=(('one', 'One'), ('two', 'Two'), ('three', 'Three'))))
json_schema = reg.get_json_schema_dict()
source = {'string_array': ['one', 'two']}
self.assertEqual(transforms.validate_object_matches_json_schema(
source, json_schema), [])
self.assertEqual(transforms.json_to_dict(source, json_schema), source)
def test_nested_dict(self):
reg = schema_fields.FieldRegistry('Test')
sub_registry = schema_fields.FieldRegistry('subregistry')
sub_registry.add_property(schema_fields.SchemaField(
'name', 'Name', 'string', description='user name'))
sub_registry.add_property(schema_fields.SchemaField(
'city', 'City', 'string', description='city name'))
reg.add_sub_registry('sub_registry', title='Sub Registry',
description='a sub-registry',
registry=sub_registry)
complaints = transforms.validate_object_matches_json_schema(
{'sub_registry': {'name': 'John Smith', 'city': 'Back East'}},
reg.get_json_schema_dict())
self.assertEqual(complaints, [])
def test_array_with_bad_members(self):
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.FieldArray(
'scalar_array', 'Scalar Array',
item_type=schema_fields.SchemaField(
'a_string', 'A String', 'string')))
complaints = transforms.validate_object_matches_json_schema(
{'scalar_array': ['foo', 123, 'bar', 456, 'baz']},
reg.get_json_schema_dict())
self.assertEqual(
complaints,
['Expected <type \'basestring\'> at Test.scalar_array[1], '
'but instead had <type \'int\'>',
'Expected <type \'basestring\'> at Test.scalar_array[3], '
'but instead had <type \'int\'>'])
def test_nested_dict_missing_items(self):
reg = schema_fields.FieldRegistry('Test')
sub_registry = schema_fields.FieldRegistry('subregistry')
sub_registry.add_property(schema_fields.SchemaField(
'name', 'Name', 'string', description='user name'))
sub_registry.add_property(schema_fields.SchemaField(
'city', 'City', 'string', description='city name'))
reg.add_sub_registry('sub_registry', title='Sub Registry',
description='a sub-registry',
registry=sub_registry)
complaints = transforms.validate_object_matches_json_schema(
{'sub_registry': {}},
reg.get_json_schema_dict())
self.assertEqual(
complaints,
['Missing mandatory value at Test.sub_registry.name',
'Missing mandatory value at Test.sub_registry.city'])
def test_array_of_dict(self):
sub_registry = schema_fields.FieldRegistry('subregistry')
sub_registry.add_property(schema_fields.SchemaField(
'name', 'Name', 'string', description='user name'))
sub_registry.add_property(schema_fields.SchemaField(
'city', 'City', 'string', description='city name'))
reg = schema_fields.FieldRegistry('Test')
reg.add_property(schema_fields.FieldArray(
'struct_array', 'Struct Array', item_type=sub_registry))
complaints = transforms.validate_object_matches_json_schema(
{'struct_array': [
{'name': 'One', 'city': 'Two'},
None,
{'name': 'Three'},
{'city': 'Four'}
]},
reg.get_json_schema_dict())
self.assertEqual(
complaints,
['Found None at Test.struct_array[1]',
'Missing mandatory value at Test.struct_array[2].city',
'Missing mandatory value at Test.struct_array[3].name'])
def reduce(user_id, values):
# Convenience for collections: Pre-load Student and Course objects.
student = None
try:
student = models.Student.get_by_user_id(user_id)
# pylint: disable=broad-except
except Exception:
common_utils.log_exception_origin()
if not student:
logging.warning(
'Student for student aggregation with user ID %s '
'was not loaded. Ignoring records for this student.', user_id)
return
params = context.get().mapreduce_spec.mapper.params
ns = params['course_namespace']
app_context = sites.get_course_index().get_app_context_for_namespace(ns)
course = courses.Course(None, app_context=app_context)
# Bundle items together into lists by collection name
event_items = collections.defaultdict(list)
for value in values:
component_name, payload = value.split(':', 1)
event_items[component_name].append(transforms.loads(payload))
# Build up per-Student aggregate by calling each component. Note that
# we call each component whether or not its mapper produced any
# output.
aggregate = {}
for component in StudentAggregateComponentRegistry.get_components():
component_name = component.get_name()
static_value = params.get(component_name)
value = {}
try:
value = component.produce_aggregate(
course, student, static_value,
event_items.get(component_name, []))
if not value:
continue
# pylint: disable=broad-except
except Exception, ex:
common_utils.log_exception_origin()
logging.critical('Student aggregation reduce function '
'component handler %s failed: %s',
component_name, str(ex))
continue
schema_name = params['schema_names'][component_name]
if schema_name not in value:
logging.critical(
'Student aggregation reduce handler %s produced '
'a dict which does not contain the top-level '
'name (%s) from its registered schema.',
component_name, schema_name)
continue
variances = transforms.validate_object_matches_json_schema(
value[schema_name], params['schemas'][component_name])
if variances:
logging.critical(
'Student aggregation reduce handler %s produced '
'a value which does not match its schema: %s',
component_name, ' '.join(variances))
continue
aggregate.update(value)
def reduce(user_id, values):
# Convenience for collections: Pre-load Student and Course objects.
student = None
try:
student = models.Student.get_by_user_id(user_id)
# pylint: disable=broad-except
except Exception:
common_utils.log_exception_origin()
if not student:
logging.warning(
'Student for student aggregation with user ID %s '
'was not loaded. Ignoring records for this student.', user_id)
return
params = context.get().mapreduce_spec.mapper.params
ns = params['course_namespace']
app_context = sites.get_course_index().get_app_context_for_namespace(
ns)
course = courses.Course(None, app_context=app_context)
# Bundle items together into lists by collection name
event_items = collections.defaultdict(list)
for value in values:
component_name, payload = value.split(':', 1)
event_items[component_name].append(transforms.loads(payload))
# Build up per-Student aggregate by calling each component. Note that
# we call each component whether or not its mapper produced any
# output.
aggregate = {}
for component in StudentAggregateComponentRegistry.get_components():
component_name = component.get_name()
static_value = params.get(component_name)
value = {}
try:
value = component.produce_aggregate(
course, student, static_value,
event_items.get(component_name, []))
if not value:
continue
# pylint: disable=broad-except
except Exception, ex:
common_utils.log_exception_origin()
logging.critical(
'Student aggregation reduce function '
'component handler %s failed: %s', component_name, str(ex))
continue
schema_name = params['schema_names'][component_name]
if schema_name not in value:
logging.critical(
'Student aggregation reduce handler %s produced '
'a dict which does not contain the top-level '
'name (%s) from its registered schema.', component_name,
schema_name)
continue
variances = transforms.validate_object_matches_json_schema(
value[schema_name], params['schemas'][component_name])
if variances:
logging.critical(
'Student aggregation reduce handler %s produced '
'a value which does not match its schema: %s',
component_name, ' '.join(variances))
continue
aggregate.update(value)
