def __init__(self, config, tracker=None, relative=True): """Construct validator instance. Args: config (dict): School config dictionary. tracker (None, optional): Description relative (bool, optional): Enforce relative ordering in validation. """ Validator.load_schemas() self.kind_to_validation_function = { kind: getattr(self, 'validate_' + kind) if hasattr(self, 'validate_' + kind) else lambda *_, **__: None for kind in Validator.KINDS } # Running monitor of validated course and section codes. self.seen = {} self.config = DotDict(config) self.config['kind'] = 'config' self.validate(self.config) self.course_code_regex = re.compile(self.config.course_code_regex) self.relative = relative if tracker is None: # Used during self-contained validation. self.tracker = Tracker() self.tracker.school = self.config.school.code self.tracker.mode = 'validating' self.tracker.start() else: self.tracker = tracker
def handle(self, *args, **options): tracker = Tracker() tracker.add_viewer(LogFormatted(options['master_log'])) tracker.cmd_options = options tracker.mode = 'digesting' tracker.start() for data_type in options['types']: for school in options['schools']: self.run(tracker, school, data_type, options) tracker.end()
def __init__(self, config, tracker=None, relative=True): """Construct validator instance. Args: config (dict): School config dictionary. tracker (None, optional): Description relative (bool, optional): Enforce relative ordering in validation. """ Validator.load_schemas() self.kind_to_validation_function = { kind: getattr(self, 'validate_' + kind) if hasattr(self, 'validate_' + kind) else lambda *_, **__: None for kind in Validator.KINDS } # Running monitor of validated course and section codes. self.seen = {} self.config = DotDict(config) self.config['kind'] = 'config' self.validate(self.config) self.course_code_regex = re.compile(self.config.course_code_regex) self.relative = relative if tracker is None: # Used during self-contained validation. self.tracker = Tracker() self.tracker.school = self.config.school.code self.tracker.mode = 'validating' self.tracker.start() else: self.tracker = tracker
def handle(self, *args, **options): """Logic of the command. Args: *args: Args of command. **options: Command options. """ tracker = Tracker() self.stat_view = StatView() tracker.add_viewer(self.stat_view) tracker.mode = 'digesting' tracker.start() for data_type in options['types']: for school in options['schools']: self.run(tracker, school, data_type, options) tracker.end()
def handle(self, *args, **options): """Logic of the command. Args: *args: Args of command. **options: Command options. """ tracker = Tracker() self.stat_view = StatView() tracker.add_viewer(self.stat_view) tracker.mode = 'digesting' tracker.start() for data_type in options['types']: for school in options['schools']: self.run(tracker, school, data_type, options) tracker.end()
def handle(self, *args, **options): """Logic of the command. Args: *args: Args of command. **options: Command options. """ tracker = Tracker() tracker.mode = 'validating' if options['display_progress_bar']: tracker.add_viewer(StatProgressBar('{valid}/{total}')) tracker.start() for parser_type in options['types']: for school in options['schools']: self.run(options, school, parser_type, tracker)
def handle(self, *args, **options): """Logic of the command. Args: *args: Args of command. **options: Command options. """ tracker = Tracker() tracker.mode = 'ingesting' self.stat_view = StatView() tracker.add_viewer(self.stat_view) if options['display_progress_bar']: tracker.add_viewer( StatProgressBar('{valid}/{total}', statistics=self.stat_view)) tracker.start() for parser_type in options['types']: for school in options['schools']: tracker.school = school self.run(SCHOOLS_MAP[school].parsers[parser_type], tracker, options, parser_type, school) tracker.end()
class Validator: """Validation engine in parsing data pipeline. Attributes: config (:obj:`DotDict`): Loaded config.json. course_code_regex (:obj:`re`): Regex to match course code. kind_to_validation_function (:obj:`dict`): Map kind to validation function defined within this class. KINDS (:obj:`set`): Kinds of objects that validator validates. relative (:obj:`bool`): Enforce relative ordering in validation. seen (:obj:`dict`): Running monitor of seen courses and sections tracker (:obj:`parsing.library.tracker.Tracker`) """ KINDS = { 'config', 'datalist', 'course', 'section', 'meeting', 'directory', 'eval', 'instructor', 'final_exam', 'textbook', 'textbook_link', } def __init__(self, config, tracker=None, relative=True): """Construct validator instance. Args: config (dict): School config dictionary. tracker (None, optional): Description relative (bool, optional): Enforce relative ordering in validation. """ Validator.load_schemas() self.kind_to_validation_function = { kind: getattr(self, 'validate_' + kind) if hasattr(self, 'validate_' + kind) else lambda *_, **__: None for kind in Validator.KINDS } # Running monitor of validated course and section codes. self.seen = {} self.config = DotDict(config) self.config['kind'] = 'config' self.validate(self.config) self.course_code_regex = re.compile(self.config.course_code_regex) self.relative = relative if tracker is None: # Used during self-contained validation. self.tracker = Tracker() self.tracker.school = self.config.school.code self.tracker.mode = 'validating' self.tracker.start() else: self.tracker = tracker @classmethod def load_schemas(cls, schema_path=None): """Load JSON validation schemas. NOTE: Will load schemas as static variable (i.e. once per definition), unless schema_path is specifically defined. Args: schema_path (None, str, optional): Override default schema_path """ if hasattr(cls, 'SCHEMAS') and schema_path is None: return if schema_path is None: schema_path = '{}/{}/library/schemas'.format( settings.BASE_DIR, settings.PARSING_MODULE) def load(kind): filepath = '{}/{}.json'.format(schema_path, kind) with open(filepath, 'r') as file: schema = json.load(file) resolved = jsonschema.RefResolver('file://{}/'.format(schema_path), schema) return (schema, resolved) cls.SCHEMAS = DotDict({kind: load(kind) for kind in cls.KINDS}) # TODO - make into a namedtuple instead @staticmethod def schema_validate(data, schema, resolver=None): """Validate data object with JSON schema alone. Args: data (dict): Data object to validate. schema: JSON schema to validate against. resolver (None, optional): JSON Schema reference resolution. Raises: jsonschema.exceptions.ValidationError: Invalid object. """ try: jsonschema.Draft4Validator(schema, resolver=resolver).validate(data) except jsonschema.exceptions.ValidationError as e: raise ValidationError(data, *e.args) # TODO - Create iter_errors from jsonschema validator # NOTE: if modifying schemas it may be prudent to catch: # jsonschema.exceptions.SchemaError # jsonschema.exceptions.RefResolutionError @staticmethod def file_to_json(path, allow_duplicates=False): """Load file pointed to by path into json object dictionary. Args: path (str): allow_duplicates (bool, optional): Allow duplicate keys in JSON. Returns: dict: JSON-compliant dictionary. """ def raise_on_duplicates(ordered_pairs): """Reject duplicate keys in dictionary.""" d = {} for k, v in ordered_pairs: if k in d: raise ValidationError("duplicate key: %r" % (k, )) d[k] = v return d with open(path, 'r') as f: if allow_duplicates: return json.load(f) return json.load(f, object_pairs_hook=raise_on_duplicates) def validate(self, data, transact=True): """Validation entry/dispatcher. Args: data (list, dict): Data to validate. """ if transact: self.transaction = SimpleNamespace(key=None, values=set()) data = DotDict(data) Validator.schema_validate(data, *Validator.SCHEMAS[data.kind]) self.kind_to_validation_function[data.kind](data) if transact and self.transaction.key: self.seen.setdefault(self.transaction.key, set()).update(self.transaction.values) def validate_self_contained(self, data_path, break_on_error=True, break_on_warning=False, output_error=None, display_progress_bar=True, master_log_path=None): """Validate JSON file as without ingestor. Args: data_path (str): Path to data file. break_on_error (bool, optional): Description break_on_warning (bool, optional): Description output_error (None, optional): Error output file path. display_progress_bar (bool, optional): Description master_log_path (None, optional): Description break_on_error (bool, optional) break_on_warning (bool, optional) display_progress_bar (bool, optional) Raises: ValidationError: Description """ data = Validator.file_to_json(data_path)['$data'] # Validator.schema_validate(data, *Validator.SCHEMAS.datalist) for obj in map(DotDict, data): try: self.validate(obj) self.tracker.stats = dict(kind=obj.kind, status='valid') except ValidationError as e: logging.exception('Validation error') if break_on_error: raise ValidationError(*e.args) except ValidationWarning as e: logging.warn(e) # warnings.warn('', e, stacklevel=2) self.tracker.stats = dict(kind=obj.kind, status='total') # TODO - this should be handled by caller self.tracker.end() def validate_course(self, course): """Validate course. Args: course (DotDict): Course object to validate. Raises: MultipleDefinitionsWarning: Course has already been validated in same session. ValidationError: Invalid course. """ if 'kind' in course and course.kind != 'course': raise ValidationError(course, 'course object must be of kind course') if ('school' in course and course.school.code != self.config.school.code): raise ValidationError(course, 'course schools does not match config') if self.course_code_regex.match(course.code) is None: raise ValidationError( course, "course code {} does not match r'{}'".format( course.code, self.config.course_code_regex)) if ('department' in course and 'code' in course.department and 'departments' in self.config): department_codes = {d.code for d in self.config.departments} if course.department.code not in department_codes: raise ValidationError( course, 'department {} is not in config.json departments'.format( course.department)) if 'homepage' in course: self.validate_website(course.homepage) for sa in course.get('same_as', []): if self.course_code_regex.match(sa) is not None: continue # raise ValidationError( # course, # "same as course code {} does not match r'{}'".format( # course.code, # self.config.course_code_regex # ) # ) if self.relative: if course.code in self.seen: raise MultipleDefinitionsWarning( course, 'multiple definitions of course {}'.format(course.code)) self.transaction.key = course.code for section in course.get('sections', []): if ('course' in section and section['course']['code'] != course.code): raise ValidationError( course, 'nested {} does not match parent {}'.format( section['course']['code'], course.code)) # NOTE: mutating dictionary section['course'] = {'code': course.code} section['kind'] = 'section' self.validate(DotDict(section), transact=False) def validate_section(self, section): """Validate section object. Args: section (DotDict): Section object to validate. Raises: MultipleDefinitionsWarning: Invalid section. ValidationError: Description """ if 'course' not in section: raise ValidationError(section, 'section doesnt define a parent course') if 'kind' in section and section.kind != 'section': raise ValidationError(section, 'section must be of kind section') if ('course' in section and self.course_code_regex.match(section.course.code) is None): raise ValidationError( section, 'course code {} does not match r\'{}\''.format( section.course.code, self.config.course_code_regex)) if 'term' in section and section.term not in self.config.terms: raise ValidationError( section, 'term {} not in config.json term list'.format(section.term)) if 'instructors' in section: db_instructor_textfield_max_size = 500 instructor_textfield = '' for instructor in section.get('instructors', []): instructor = DotDict(instructor) if isinstance(instructor.name, basestring): instructor_textfield += instructor.name elif isinstance(instructor.name, dict): instructor_textfield += '{} {}'.format( instructor.name.first, instructor.name.last) db_instructor_textfield_size = len(instructor_textfield) if db_instructor_textfield_size > db_instructor_textfield_max_size: raise ValidationError( section, 'db field too small for comma-joined instructor names') for instructor in section.get('instructors', []): self.validate_instructor(instructor) if 'final_exam' in section: if ('course' in section.final_exam and section.final_exam.course.code != section.course.code): raise ValidationError( section, 'final exam course {} doesnt match course code {}'.format( section.final_exam.course.code, section.course.code)) if ('section' in section.final_exam and section.final_exam.section.code != section.code): raise ValidationError( section, 'final exam section {} doesnt match section {}'.format( section.final_exam.section.code, section.code)) # final_exam['course'] = section.course # final_exam['section'] = {'code': section.code} # self.validate_final_exam(section.final_exam) if self.relative: if section.course.code not in self.seen and self.transaction.key != section.course.code: print(self.seen) raise ValidationError( 'course code {} isnt defined'.format(section.course.code), section) elif ((section.code, section.year, section.term) in self.seen.get(section.course.code, set()) | self.transaction.values): raise MultipleDefinitionsWarning( section, 'multiple defs for {} {} - {} already defined'.format( section.course.code, section.code, section.year)) self.transaction.key = section.course.code self.transaction.values.add( (section.code, section.year, section.term)) for meeting in section.get('meetings', []): meeting = DotDict(meeting) if ('course' in meeting and meeting.course.code != section.course.code): raise ValidationError( section, 'course code {} in meeting doesnt match parent section \ course code {}'.format(meeting.course.code, section.course.code)) if 'section' in meeting and meeting.section.code != section.code: raise ValidationError( section, 'section code {} in nested meeting doesnt match parent \ section code {}'.format(meeting.section.code, section.code)) # NOTE: mutating obj meeting['course'] = section.course meeting['section'] = { 'code': section.code, 'year': section.year, 'term': section.term } meeting['kind'] = 'meeting' self.validate(DotDict(meeting), transact=False) if 'textbooks' in section: for textbook in section.textbooks: self.validate_textbook_link(textbook) def validate_meeting(self, meeting): """Validate meeting object. Args: meeting (DotDict): Meeting object to validate. Raises: ValidationError: Invalid meeting. ValidationWarning: Description """ if 'kind' in meeting and meeting.kind != 'meeting': raise ValidationError(meeting, 'meeting object must be kind instructor') if ('course' in meeting and self.course_code_regex.match(meeting.course.code) is None): raise ValidationError( meeting, 'course code {} does not match regex \'{}\''.format( meeting.course.code, self.config.course_code_regex)) if 'time' in meeting: try: self.validate_time_range(meeting.time.start, meeting.time.end) except (ValidationError, ValidationWarning) as e: message = 'meeting for {} {}, '.format(meeting.course.code, meeting.section.code) if isinstance(e, ValidationError): raise ValidationError(message, *e.args) raise ValidationWarning(message, *e.args) if 'location' in meeting: try: self.validate_location(meeting.location) except ValidationError as e: message = 'meeting for {} {}, '.format(meeting.course.code, meeting.section.code) raise ValidationError(message, *e.args) if not self.relative: return if 'course' in meeting and meeting.course.code not in self.seen and self.transaction is None: raise ValidationError( meeting, 'course code {} isnt defined'.format(meeting.course.code)) if 'section' not in meeting: return if (meeting.section.code, meeting.section.year, meeting.section.term) not in self.seen.get( meeting.course.code, set()) | self.transaction.values: raise ValidationError( meeting, 'section {} isnt defined'.format(meeting.section.code)) def validate_eval(self, course_eval): """Validate evaluation object. Args: course_eval (DotDict): Evaluation to validate. Raises: ValidationError: Invalid evaulation. """ if self.course_code_regex.match(course_eval.course.code) is None: raise ValidationError( course_eval, "course code {} does not match r'{}'".format( course_eval.course.code, self.config.course_code_regex)) def validate_instructor(self, instructor): """Validate instructor object. Args: instructor (DotDict): Instructor object to validate. Raises: ValidationError: Invalid instructor. """ if 'kind' in instructor and instructor.kind != 'instructor': raise ValidationError( instructor, 'instructor object must be of kind instructor') for class_ in instructor.get('classes', []): if ('course' in class_ and self.course_code_regex.match(class_.course.code) is None): raise ValidationError( instructor, 'course code {} does not match given regex {}'.format( class_.course.code, self.config.course_code_regex)) if 'department' in instructor and 'departments' in self.config: dept_codes = {d.code for d in self.config.departments} if instructor.department not in dept_codes: raise ValidationError( instructor, 'department {} not listed in config.json'.format( instructor.department)) if 'homepage' in instructor: try: self.validate_homepage(instructor.homepage) except ValidationError as e: message = 'instructor {} office, {}'.format(instructor.name) raise ValidationError(message, *e.args) if 'office' in instructor: try: if 'location' in instructor.office: self.validate_location(instructor.office.location) for office_hour in instructor.office.get('hours', []): self.validate_meeting(office_hour) except ValidationError as e: message = 'instructor {} office, {}'.format(instructor.name) raise ValidationError(message, *e.args) def validate_final_exam(self, final_exam): """Validate final exam. NOTE: currently unused. Args: final_exam (DotDict): Final Exam object to validate. Raises: ValidationError: Invalid final exam. """ if 'kind' in final_exam and final_exam.kind != 'final_exam': raise ValidationError( final_exam, 'final_exam object must be of kind "final_exam"') try: self.validate_meeting(final_exam.meeting) except ValidationError as e: raise ValidationError(final_exam, *e.args) def validate_textbook_link(self, textbook_link): """Validate textbook link. Args: textbook_link (DotDict): Textbook link object to validate. Raises: ValidationError: Invalid textbook link. """ if 'course' not in textbook_link: return if self.course_code_regex.match(textbook_link.course.code) is not None: return raise ValidationError( textbook_link, 'textbook_link course code doent match course code regex') def validate_location(self, location): """Validate location. Args: location (DotDict): Location object to validate. Raises: ValidationWarning: Invalid location. """ if 'campus' in location and 'campuses' in self.config: if location.campus not in self.config.campuses: raise ValidationWarning( location, 'campus {} not in config'.format(location.campus), ) if 'building' in location and 'buildings' in self.config: if location.building not in self.config.buildings: raise ValidationWarning( location, 'building {} not in config'.format(location.building), ) @staticmethod def validate_website(url): """Validate url by sending HEAD request and analyzing response. Args: url (str): URL to validate. Raises: ValidationError: URL is invalid. """ c = httplib.HTTPConnection(url) c.request('HEAD', '') # NOTE: 200 - good status # 301 - redirected if c.getresponse().status == 200 or c.getresponse().status == 301: return raise ValidationError(url, 'invalid website w/url "%s"'.format(url)) def validate_time_range(self, start, end): """Validate start time and end time. There exists an unhandled case if the end time is midnight. Args: start (str): Start time. end (str): End time. Raises: ValidationError: Time range is invalid. """ try: start, end = map(dparser.parse, [start, end]) except ValueError: raise ValidationError('invalid time format {}-{}'.format( start, end)) if start > end: raise ValidationError('start {} > end {}'.format(start, end)) elif start == end: pass # TODO - this should be reported # raise ValidationWarning('start {} = end {}'.format(start, end)) # NOTE: there exists an unhandled case if the end time is midnight. def validate_directory(self, directory): """Validate directory. Args: directory (str, dict): Directory to validate. May be either path or object. Raises: ValidationError: encapsulated IOError """ if isinstance(directory, str): try: name = directory directory = dir_to_dict(directory) directory['name'] = name except IOError as e: raise ValidationError(str(e)) Validator.schema_validate(directory, *Validator.SCHEMAS.directory)
def handle(self, *args, **options): """Logic of the command. Args: *args: Args of command. **options: Command options. """ tracker = Tracker() tracker.mode = "ingesting" self.stat_view = StatView() tracker.add_viewer(self.stat_view) if options["display_progress_bar"]: tracker.add_viewer( StatProgressBar("{valid}/{total}", statistics=self.stat_view) ) tracker.start() for parser_type in options["types"]: for school in options["schools"]: tracker.school = school try: parsing = __import__( "parsing.schools.{school}.{parser_type}".format( school=school, parser_type=parser_type ) ) parser = eval( "parsing.schools.{school}.{parser_type}.Parser".format( school=school, parser_type=parser_type ) ) self.run(parser, tracker, options, parser_type, school) except ImportError: logging.exception("Invalid parser") continue tracker.end()
def handle(self, *args, **options): """Logic of the command. Args: *args: Args of command. **options: Command options. """ tracker = Tracker() tracker.cmd_options = options tracker.add_viewer(LogFormatted(options['master_log'])) tracker.mode = 'ingesting' if options['display_progress_bar']: tracker.add_viewer(ProgressBar('{valid}/{total}')) tracker.start() for data_type in options['types']: for school in options['schools']: tracker.school = school # TODO - remove after deprecation if school not in parsers[data_type]: old_map = { 'textbooks': textbook_parsers, 'courses': course_parsers, 'evals': eval_parsers, } self.old_parser(old_map[data_type][school], school) continue # END - remove after deprecation self.run(parsers[data_type][school], tracker, options, data_type, school) tracker.end()
def handle(self, *args, **options): """Logic of the command. Args: *args: Args of command. **options: Command options. """ tracker = Tracker() tracker.mode = 'ingesting' self.stat_view = StatView() tracker.add_viewer(self.stat_view) if options['display_progress_bar']: tracker.add_viewer(StatProgressBar('{valid}/{total}', statistics=self.stat_view)) tracker.start() for parser_type in options['types']: for school in options['schools']: tracker.school = school self.run(SCHOOLS_MAP[school].parsers[parser_type], tracker, options, parser_type, school) tracker.end()
class Validator: """Validation engine in parsing data pipeline. Attributes: config (:obj:`DotDict`): Loaded config.json. course_code_regex (:obj:`re`): Regex to match course code. kind_to_validation_function (:obj:`dict`): Map kind to validation function defined within this class. KINDS (:obj:`set`): Kinds of objects that validator validates. relative (:obj:`bool`): Enforce relative ordering in validation. seen (:obj:`dict`): Running monitor of seen courses and sections tracker (:obj:`parsing.library.tracker.Tracker`) """ KINDS = { 'config', 'datalist', 'course', 'section', 'meeting', 'directory', 'eval', 'instructor', 'final_exam', 'textbook', 'textbook_link', } def __init__(self, config, tracker=None, relative=True): """Construct validator instance. Args: config (dict): School config dictionary. tracker (None, optional): Description relative (bool, optional): Enforce relative ordering in validation. """ Validator.load_schemas() self.kind_to_validation_function = { kind: getattr(self, 'validate_' + kind) if hasattr(self, 'validate_' + kind) else lambda *_, **__: None for kind in Validator.KINDS } # Running monitor of validated course and section codes. self.seen = {} self.config = DotDict(config) self.config['kind'] = 'config' self.validate(self.config) self.course_code_regex = re.compile(self.config.course_code_regex) self.relative = relative if tracker is None: # Used during self-contained validation. self.tracker = Tracker() self.tracker.school = self.config.school.code self.tracker.mode = 'validating' self.tracker.start() else: self.tracker = tracker @classmethod def load_schemas(cls, schema_path=None): """Load JSON validation schemas. NOTE: Will load schemas as static variable (i.e. once per definition), unless schema_path is specifically defined. Args: schema_path (None, str, optional): Override default schema_path """ if hasattr(cls, 'SCHEMAS') and schema_path is None: return if schema_path is None: schema_path = '{}/{}/library/schemas'.format( settings.BASE_DIR, settings.PARSING_MODULE ) def load(kind): filepath = '{}/{}.json'.format(schema_path, kind) with open(filepath, 'r') as file: schema = json.load(file) resolved = jsonschema.RefResolver( 'file://{}/'.format(schema_path), schema ) return (schema, resolved) cls.SCHEMAS = DotDict({ kind: load(kind) for kind in cls.KINDS }) # TODO - make into a namedtuple instead @staticmethod def schema_validate(data, schema, resolver=None): """Validate data object with JSON schema alone. Args: data (dict): Data object to validate. schema: JSON schema to validate against. resolver (None, optional): JSON Schema reference resolution. Raises: jsonschema.exceptions.ValidationError: Invalid object. """ try: jsonschema.Draft4Validator(schema, resolver=resolver).validate(data) except jsonschema.exceptions.ValidationError as e: raise ValidationError(data, *e.args) # TODO - Create iter_errors from jsonschema validator # NOTE: if modifying schemas it may be prudent to catch: # jsonschema.exceptions.SchemaError # jsonschema.exceptions.RefResolutionError @staticmethod def file_to_json(path, allow_duplicates=False): """Load file pointed to by path into json object dictionary. Args: path (str): allow_duplicates (bool, optional): Allow duplicate keys in JSON. Returns: dict: JSON-compliant dictionary. """ def raise_on_duplicates(ordered_pairs): """Reject duplicate keys in dictionary.""" d = {} for k, v in ordered_pairs: if k in d: raise ValidationError("duplicate key: %r" % (k,)) d[k] = v return d with open(path, 'r') as f: if allow_duplicates: return json.load(f) return json.load(f, object_pairs_hook=raise_on_duplicates) def validate(self, data, transact=True): """Validation entry/dispatcher. Args: data (list, dict): Data to validate. """ if transact: self.transaction = SimpleNamespace(key=None, values=set()) data = DotDict(data) Validator.schema_validate(data, *Validator.SCHEMAS[data.kind]) self.kind_to_validation_function[data.kind](data) if transact and self.transaction.key: self.seen.setdefault(self.transaction.key, set()).update(self.transaction.values) def validate_self_contained(self, data_path, break_on_error=True, break_on_warning=False, output_error=None, display_progress_bar=True, master_log_path=None): """Validate JSON file as without ingestor. Args: data_path (str): Path to data file. break_on_error (bool, optional): Description break_on_warning (bool, optional): Description output_error (None, optional): Error output file path. display_progress_bar (bool, optional): Description master_log_path (None, optional): Description break_on_error (bool, optional) break_on_warning (bool, optional) display_progress_bar (bool, optional) Raises: ValidationError: Description """ data = Validator.file_to_json(data_path)['$data'] # Validator.schema_validate(data, *Validator.SCHEMAS.datalist) for obj in map(DotDict, data): try: self.validate(obj) self.tracker.stats = dict(kind=obj.kind, status='valid') except ValidationError as e: logging.exception('Validation error') if break_on_error: raise ValidationError(*e.args) except ValidationWarning as e: logging.warn(e) # warnings.warn('', e, stacklevel=2) self.tracker.stats = dict(kind=obj.kind, status='total') # TODO - this should be handled by caller self.tracker.end() def validate_course(self, course): """Validate course. Args: course (DotDict): Course object to validate. Raises: MultipleDefinitionsWarning: Course has already been validated in same session. ValidationError: Invalid course. """ if 'kind' in course and course.kind != 'course': raise ValidationError(course, 'course object must be of kind course') if ('school' in course and course.school.code != self.config.school.code): raise ValidationError(course, 'course schools does not match config') if self.course_code_regex.match(course.code) is None: raise ValidationError( course, "course code {} does not match r'{}'".format( course.code, self.config.course_code_regex ) ) if ('department' in course and 'code' in course.department and 'departments' in self.config): department_codes = {d.code for d in self.config.departments} if course.department.code not in department_codes: raise ValidationError( course, 'department {} is not in config.json departments'.format( course.department) ) if 'homepage' in course: self.validate_website(course.homepage) for sa in course.get('same_as', []): if self.course_code_regex.match(sa) is not None: continue # raise ValidationError( # course, # "same as course code {} does not match r'{}'".format( # course.code, # self.config.course_code_regex # ) # ) if self.relative: if course.code in self.seen: raise MultipleDefinitionsWarning( course, 'multiple definitions of course {}'.format(course.code) ) self.transaction.key = course.code for section in course.get('sections', []): if ('course' in section and section['course']['code'] != course.code): raise ValidationError( course, 'nested {} does not match parent {}'.format( section['course']['code'], course.code ) ) # NOTE: mutating dictionary section['course'] = {'code': course.code} section['kind'] = 'section' self.validate(DotDict(section), transact=False) def validate_section(self, section): """Validate section object. Args: section (DotDict): Section object to validate. Raises: MultipleDefinitionsWarning: Invalid section. ValidationError: Description """ if 'course' not in section: raise ValidationError(section, 'section doesnt define a parent course') if 'kind' in section and section.kind != 'section': raise ValidationError(section, 'section must be of kind section') if ('course' in section and self.course_code_regex.match(section.course.code) is None): raise ValidationError( section, 'course code {} does not match r\'{}\''.format( section.course.code, self.config.course_code_regex ) ) if 'term' in section and section.term not in self.config.terms: raise ValidationError( section, 'term {} not in config.json term list'.format(section.term) ) if 'instructors' in section: db_instructor_textfield_max_size = 500 instructor_textfield = '' for instructor in section.get('instructors', []): instructor = DotDict(instructor) if isinstance(instructor.name, basestring): instructor_textfield += instructor.name elif isinstance(instructor.name, dict): instructor_textfield += '{} {}'.format(instructor.name.first, instructor.name.last) db_instructor_textfield_size = len(instructor_textfield) if db_instructor_textfield_size > db_instructor_textfield_max_size: raise ValidationError( section, 'db field too small for comma-joined instructor names' ) for instructor in section.get('instructors', []): self.validate_instructor(instructor) if 'final_exam' in section: if ('course' in section.final_exam and section.final_exam.course.code != section.course.code): raise ValidationError( section, 'final exam course {} doesnt match course code {}'.format( section.final_exam.course.code, section.course.code ) ) if ('section' in section.final_exam and section.final_exam.section.code != section.code): raise ValidationError( section, 'final exam section {} doesnt match section {}'.format( section.final_exam.section.code, section.code ) ) # final_exam['course'] = section.course # final_exam['section'] = {'code': section.code} # self.validate_final_exam(section.final_exam) if self.relative: if section.course.code not in self.seen and self.transaction.key != section.course.code: print(self.seen) raise ValidationError( 'course code {} isnt defined'.format(section.course.code), section ) elif ((section.code, section.year, section.term) in self.seen.get(section.course.code, set()) | self.transaction.values): raise MultipleDefinitionsWarning( section, 'multiple defs for {} {} - {} already defined'.format( section.course.code, section.code, section.year ) ) self.transaction.key = section.course.code self.transaction.values.add((section.code, section.year, section.term)) for meeting in section.get('meetings', []): meeting = DotDict(meeting) if ('course' in meeting and meeting.course.code != section.course.code): raise ValidationError( section, 'course code {} in meeting doesnt match parent section \ course code {}'.format( meeting.course.code, section.course.code ) ) if 'section' in meeting and meeting.section.code != section.code: raise ValidationError( section, 'section code {} in nested meeting doesnt match parent \ section code {}'.format( meeting.section.code, section.code ) ) # NOTE: mutating obj meeting['course'] = section.course meeting['section'] = { 'code': section.code, 'year': section.year, 'term': section.term } meeting['kind'] = 'meeting' self.validate(DotDict(meeting), transact=False) if 'textbooks' in section: for textbook in section.textbooks: self.validate_textbook_link(textbook) def validate_meeting(self, meeting): """Validate meeting object. Args: meeting (DotDict): Meeting object to validate. Raises: ValidationError: Invalid meeting. ValidationWarning: Description """ if 'kind' in meeting and meeting.kind != 'meeting': raise ValidationError(meeting, 'meeting object must be kind instructor') if ('course' in meeting and self.course_code_regex.match(meeting.course.code) is None): raise ValidationError( meeting, 'course code {} does not match regex \'{}\''.format( meeting.course.code, self.config.course_code_regex ) ) if 'time' in meeting: try: self.validate_time_range(meeting.time.start, meeting.time.end) except (ValidationError, ValidationWarning) as e: message = 'meeting for {} {}, '.format( meeting.course.code, meeting.section.code ) if isinstance(e, ValidationError): raise ValidationError(message, *e.args) raise ValidationWarning(message, *e.args) if 'location' in meeting: try: self.validate_location(meeting.location) except ValidationError as e: message = 'meeting for {} {}, '.format( meeting.course.code, meeting.section.code ) raise ValidationError(message, *e.args) if not self.relative: return if 'course' in meeting and meeting.course.code not in self.seen and self.transaction is None: raise ValidationError( meeting, 'course code {} isnt defined'.format(meeting.course.code) ) if 'section' not in meeting: return if (meeting.section.code, meeting.section.year, meeting.section.term) not in self.seen.get(meeting.course.code, set()) | self.transaction.values: raise ValidationError( meeting, 'section {} isnt defined'.format(meeting.section.code) ) def validate_eval(self, course_eval): """Validate evaluation object. Args: course_eval (DotDict): Evaluation to validate. Raises: ValidationError: Invalid evaulation. """ if self.course_code_regex.match(course_eval.course.code) is None: raise ValidationError( course_eval, "course code {} does not match r'{}'".format( course_eval.course.code, self.config.course_code_regex ) ) def validate_instructor(self, instructor): """Validate instructor object. Args: instructor (DotDict): Instructor object to validate. Raises: ValidationError: Invalid instructor. """ if 'kind' in instructor and instructor.kind != 'instructor': raise ValidationError( instructor, 'instructor object must be of kind instructor' ) for class_ in instructor.get('classes', []): if ('course' in class_ and self.course_code_regex.match(class_.course.code) is None): raise ValidationError( instructor, 'course code {} does not match given regex {}'.format( class_.course.code, self.config.course_code_regex ) ) if 'department' in instructor and 'departments' in self.config: dept_codes = {d.code for d in self.config.departments} if instructor.department not in dept_codes: raise ValidationError( instructor, 'department {} not listed in config.json'.format( instructor.department ) ) if 'homepage' in instructor: try: self.validate_homepage(instructor.homepage) except ValidationError as e: message = 'instructor {} office, {}'.format(instructor.name) raise ValidationError(message, *e.args) if 'office' in instructor: try: if 'location' in instructor.office: self.validate_location(instructor.office.location) for office_hour in instructor.office.get('hours', []): self.validate_meeting(office_hour) except ValidationError as e: message = 'instructor {} office, {}'.format(instructor.name) raise ValidationError(message, *e.args) def validate_final_exam(self, final_exam): """Validate final exam. NOTE: currently unused. Args: final_exam (DotDict): Final Exam object to validate. Raises: ValidationError: Invalid final exam. """ if 'kind' in final_exam and final_exam.kind != 'final_exam': raise ValidationError( final_exam, 'final_exam object must be of kind "final_exam"' ) try: self.validate_meeting(final_exam.meeting) except ValidationError as e: raise ValidationError(final_exam, *e.args) def validate_textbook_link(self, textbook_link): """Validate textbook link. Args: textbook_link (DotDict): Textbook link object to validate. Raises: ValidationError: Invalid textbook link. """ if 'course' not in textbook_link: return if self.course_code_regex.match(textbook_link.course.code) is not None: return raise ValidationError( textbook_link, 'textbook_link course code doent match course code regex' ) def validate_location(self, location): """Validate location. Args: location (DotDict): Location object to validate. Raises: ValidationWarning: Invalid location. """ if 'campus' in location and 'campuses' in self.config: if location.campus not in self.config.campuses: raise ValidationWarning( location, 'campus {} not in config'.format(location.campus), ) if 'building' in location and 'buildings' in self.config: if location.building not in self.config.buildings: raise ValidationWarning( location, 'building {} not in config'.format(location.building), ) @staticmethod def validate_website(url): """Validate url by sending HEAD request and analyzing response. Args: url (str): URL to validate. Raises: ValidationError: URL is invalid. """ c = httplib.HTTPConnection(url) c.request('HEAD', '') # NOTE: 200 - good status # 301 - redirected if c.getresponse().status == 200 or c.getresponse().status == 301: return raise ValidationError(url, 'invalid website w/url "%s"'.format(url)) def validate_time_range(self, start, end): """Validate start time and end time. There exists an unhandled case if the end time is midnight. Args: start (str): Start time. end (str): End time. Raises: ValidationError: Time range is invalid. """ try: start, end = map(dparser.parse, [start, end]) except ValueError: raise ValidationError('invalid time format {}-{}'.format(start, end)) if start > end: raise ValidationError('start {} > end {}'.format(start, end)) elif start == end: pass # TODO - this should be reported # raise ValidationWarning('start {} = end {}'.format(start, end)) # NOTE: there exists an unhandled case if the end time is midnight. def validate_directory(self, directory): """Validate directory. Args: directory (str, dict): Directory to validate. May be either path or object. Raises: ValidationError: encapsulated IOError """ if isinstance(directory, str): try: name = directory directory = dir_to_dict(directory) directory['name'] = name except IOError as e: raise ValidationError(str(e)) Validator.schema_validate(directory, *Validator.SCHEMAS.directory)
class Validator: """Validation engine in parsing data pipeline. Attributes: config (DotDict): Loaded config.json. course_code_regex (re): Regex to match course code. kind_to_validation_function (dict): Map kind to validation function defined within this class. KINDS (set): Kinds of objects that validator validates. relative (bool): Enforce relative ordering in validation. seen (dict): Running monitor of seen courses and sections tracker (parsing.parsing_library.tracker.Tracker): Tracker. """ KINDS = { 'config', 'datalist', 'course', 'section', 'meeting', 'directory', 'eval', 'instructor', 'final_exam', 'textbook', 'textbook_link', } def __init__(self, config_path, tracker=None, relative=True): """Construct validator instance. Args: config_path (str): School config file path. tracker (None, optional): Description relative (bool, optional): Enforce relative ordering in validation. """ Validator.load_schemas() self.kind_to_validation_function = { kind: getattr(self, 'validate_' + kind) if hasattr(self, 'validate_' + kind) else lambda *_, **__: None for kind in Validator.KINDS } self.config = DotDict(Validator.file_to_json(config_path)) self.config['kind'] = 'config' self.validate(self.config) self.course_code_regex = re.compile(self.config.course_code_regex) self.relative = relative # Running monitor of validated course and section codes. self.seen = {} if tracker is None: # Used during self-contained validation. self.tracker = Tracker() self.tracker.school = self.config.school.code self.tracker.mode = 'validating' self.tracker.start() else: self.tracker = tracker @classmethod def load_schemas(cls, schema_path=None): """Load JSON validation schemas. NOTE: Will load schemas as static variable (i.e. once per definition), unless schema_path is specifically defined. Args: schema_path (None, str, optional): Override default schema_path """ if hasattr(cls, 'SCHEMAS') and schema_path is None: return if schema_path is None: schema_path = '{}/{}/library/schemas'.format( settings.BASE_DIR, settings.PARSING_DIR) def load(kind): filepath = '{}/{}.json'.format(schema_path, kind) with open(filepath, 'r') as file: schema = json.load(file) resolved = jsonschema.RefResolver('file://{}/'.format(schema_path), schema) return (schema, resolved) cls.SCHEMAS = DotDict({kind: load(kind) for kind in cls.KINDS}) @staticmethod def schema_validate(data, schema, resolver=None): """Validate data object with JSON schema alone. Args: data (dict): Data object to validate schema (TYPE): JSON schema to validate against. resolver (None, optional): JSON Schema reference resolution. Raises: jsonschema.exceptions.ValidationError: Invalid object. """ jsonschema.Draft4Validator(schema, resolver=resolver).validate(data) # TODO - Create iter_errors from jsonschema validator # NOTE: if modifying schemas it may be prudent to catch: # jsonschema.exceptions.SchemaError # jsonschema.exceptions.RefResolutionError @staticmethod def file_to_json(path, allow_duplicates=False): """Load file pointed to by path into json object dictionary. Args: path (str): allow_duplicates (bool, optional): Allow duplicate keys in JSON. Returns: dict: JSON-compliant dictionary. """ def raise_on_duplicates(ordered_pairs): """Reject duplicate keys in dictionary.""" d = {} for k, v in ordered_pairs: if k in d: raise JsonValidationError("duplicate key: %r" % (k, )) d[k] = v return d with open(path, 'r') as f: if allow_duplicates: return json.load(f) return json.load(f, object_pairs_hook=raise_on_duplicates) def validate(self, data): """Validation entry/dispatcher. Args: data (list, dict): Data to validate. """ for obj in make_list(data): obj = DotDict(obj) Validator.schema_validate(obj, *Validator.SCHEMAS[obj.kind]) self.kind_to_validation_function[obj.kind](obj) def validate_self_contained(self, data_path, break_on_error=True, break_on_warning=False, output_error=None, display_progress_bar=True, master_log_path=None): """Validate JSON file as without ingestor. Args: data_path (str): Path to data file. break_on_error (bool, optional) break_on_warning (bool, optional) output_error (None, optional): Error output file path. display_progress_bar (bool, optional) Raises: e: TODO """ # TODO - iter errors and catch exceptions within method if display_progress_bar: self.tracker.add_viewer(ProgressBar('{total}')) logger = Logger(errorfile=output_error) try: # self.validate_directory(directory) data = Validator.file_to_json(data_path)['$data'] Validator.schema_validate(data, *Validator.SCHEMAS.datalist) except (JsonValidationError, json.scanner.JSONDecodeError) as e: logger.log(e) raise e # fatal error, cannot continue for obj in data: obj = DotDict(obj) try: self.kind_to_validation_function[obj.kind](obj) self.tracker.status = dict(kind=obj.kind, status='valid') except JsonValidationError as e: logger.log(e) if break_on_error: raise e except JsonValidationWarning as e: logger.log(e) if break_on_warning: raise e self.tracker.status = dict(kind=obj.kind, status='total') self.tracker.end() def validate_course(self, course): """Validate course. Args: course (DotDict): Course object to validate. Raises: JsonDuplicationWarning: TODO JsonValidationError: Invalid course. """ if 'kind' in course and course.kind != 'course': raise JsonValidationError('course object must be of kind course', course) if self.course_code_regex.match(course.code) is None: raise JsonValidationError( "course code {} does not match r'{}'".format( course.code, self.config.course_code_regex), course) if ('department' in course and 'code' in course.department and 'departments' in self.config): department_codes = {d.code for d in self.config.departments} if course.department.code not in department_codes: raise JsonValidationError( 'department {} is not in config.json departments'.format( course.department), course) if 'homepage' in course: self.validate_website(course.homepage) for section in course.get('sections', []): if 'course' in section and section.course.code != course.code: raise JsonValidationError( 'nested {} does not match parent {}'.format( section.course.code, course.code), course) # NOTE: mutating dictionary section.course = {'code': course.code} self.validate_section(section) if not self.relative: return if course.code in self.seen: raise JsonDuplicationWarning( 'multiple definitions of course {}'.format(course.code), course) self.seen.setdefault(course.code, {}) def validate_section(self, section): """Validate section object. Args: section (DotDict): Section object to validate. Raises: JsonDuplicationWarning: TODO JsonValidationError: Invalid section. """ if 'course' not in section: raise JsonValidationError('section doesnt define a parent course', section) if 'kind' in section and section.kind != 'section': raise JsonValidationError('section must be of kind "section"', section) if ('course' in section and self.course_code_regex.match(section.course.code) is None): raise JsonValidationError( 'course code {} does not match r\'{}\''.format( section.course.code, self.config.course_code_regex), section) if 'term' in section and section.term not in self.config.terms: raise JsonValidationError( 'term {} not in config.json term list'.format(section.term), section) if 'instructors' in section: db_instructor_textfield_size = 500 if len(', '.join(instructor['name'] for instructor in section. instructors)) > db_instructor_textfield_size: raise JsonValidationError( 'db field too small for comma-joined instructor names', section) for instructor in section.get('instructors', []): self.validate_instructor(instructor) if 'final_exam' in section: if ('course' in section.final_exam and section.final_exam.course.code != section.course.code): raise JsonValidationError( 'final exam course {} doesnt match course code {}'.format( section.final_exam.course.code, section.course.code), section) if ('section' in section.final_exam and section.final_exam.section.code != section.code): raise JsonValidationError( 'final exam section {} doesnt match section {}'.format( section.final_exam.section.code, section.code), section) # final_exam['course'] = section.course # final_exam['section'] = {'code': section.code} # self.validate_final_exam(section.final_exam) for meeting in section.get('meetings', []): if ('course' in meeting and meeting.course.code != section.course.code): raise JsonValidationError( 'course code {} in meeting doesnt match parent section \ course code {}'.format(meeting.course.code, section.course.code), section) if 'section' in meeting and meeting.section.code != section.code: raise JsonValidationError( 'section code {} in nested meeting doesnt match parent \ section code {}'.format(meeting.section.code, section.code), section) # NOTE: mutating obj meeting.course = section.course meeting.section = {'code': section.code} self.validate_meeting(meeting) if 'textbooks' in section: for textbook in section.textbooks: self.validate_textbook_link(textbook) if not self.relative: return if section.course.code not in self.seen: raise JsonValidationError( 'course code {} isnt defined'.format(section.course.code), section) elif (section.code in self.seen[section.course.code] and section.year in self.seen[section.course.code][section.code] and section.term in self.seen[section.course.code][section.code][section.year]): raise JsonDuplicationWarning( 'multiple defs for {} {} - {} already defined'.format( section.course.code, section.code, section.year), section) update(self.seen[section.course.code], {section.code: { section.year: section.term }}) def validate_meeting(self, meeting): """Validate meeting object. Args: meeting (DotDict): Meeting object to validate. Raises: e: TODO JsonValidationError: Invalid meeting. """ if 'kind' in meeting and meeting.kind != 'meeting': raise JsonValidationError('meeting object must be kind instructor', meeting) if ('course' in meeting and self.course_code_regex.match(meeting.course.code) is None): raise JsonValidationError( 'course code {} does not match regex \'{}\''.format( meeting.course.code, self.config.course_code_regex), meeting) if 'time' in meeting: try: self.validate_time_range(meeting.time.start, meeting.time.end) except (JsonValidationError, JsonValidationWarning) as e: e.message = 'meeting for {} {}, '.format( meeting.course.code, meeting.section.code) + e.message raise e if 'location' in meeting: try: self.validate_location(meeting.location) except JsonValidationError as e: e.message = 'meeting for {} {}, '.format( meeting.course.code, meeting.section.code) + e.message raise e if not self.relative: return if meeting.course.code not in self.seen: raise JsonValidationError( 'course code {} isnt defined'.format(meeting.course.code), meeting) if meeting.section.code not in self.seen[meeting.course.code]: raise JsonValidationError( 'section {} isnt defined'.format(meeting.section.code), meeting) def validate_eval(self, course_eval): """Validate evaluation object. Args: course_eval (DotDict): Evaluation to validate. Raises: JsonValidationError: Invalid evaulation. """ if not isinstance(course_eval, DotDict): course_eval = DotDict(course_eval) if self.course_code_regex.match(course_eval.course.code) is None: raise JsonValidationError( "course code {} does not match r'{}'".format( course_eval.course.code, self.config.course_code_regex), course_eval) def validate_instructor(self, instructor): """Validate instructor object. Args: instructor (DotDict): Instructor object to validate. Raises: e: TODO JsonValidationError: Invalid instructor. """ if 'kind' in instructor and instructor.kind != 'instructor': raise JsonValidationError( 'instructor object must be of kind instructor', instructor) for class_ in instructor.get('classes', []): if ('course' in class_ and self.course_code_regex.match(class_.course.code) is None): raise JsonValidationError( 'course code {} does not match given regex {}'.format( class_.course.code, self.config.course_code_regex), instructor) if 'department' in instructor and 'departments' in self.config: dept_codes = {d.code for d in self.config.departments} if instructor.department not in dept_codes: raise JsonValidationError( 'department {} not listed in config.json'.format( instructor.department), instructor) if 'homepage' in instructor: try: self.validate_homepage(instructor.homepage) except JsonValidationError as e: e.message = '@instructor {} office, {}'.format( instructor.name, e.message) raise e if 'office' in instructor: try: if 'location' in instructor.office: self.validate_location(instructor.office.location) for office_hour in instructor.office.get('hours', []): self.validate_meeting(office_hour) except JsonValidationError as e: e.message = '@instructor {} office, {}'.format( instructor.name, e.message) raise e def validate_final_exam(self, final_exam): """Validate final exam. Args: final_exam (DotDict): Final Exam object to validate. Raises: e: TODO JsonValidationError: Invalid Final Exam. """ if 'kind' in final_exam and final_exam.kind != 'final_exam': raise JsonValidationError( 'final_exam object must be of kind "final_exam"', final_exam) try: self.validate_meeting(final_exam.meeting) except JsonValidationError as e: e.message = '@final_exam ' + e.message raise e def validate_textbook_link(self, textbook_link): """Validate textbook link. Args: textbook_link (DotDict): Textbook link object to validate. Raises: JsonValidationError: Invalid textbook link. """ if 'course' not in textbook_link: return if self.course_code_regex.match(textbook_link.course.code) is not None: return raise JsonValidationError( 'textbook_link course code doent match course code regex', textbook_link) def validate_location(self, location): """Validate location. Args: location (DotDict): Location object to validate. Raises: JsonValidationWarning: Invalid location. """ if 'campus' in location and 'campuses' in self.config: if location.campus not in self.config.campuses: raise JsonValidationWarning( 'campus {} not in config'.format(location.campus), location) if 'building' in location and 'buildings' in self.config: if location.building not in self.config.buildings: raise JsonValidationWarning( 'building {} not in config'.format(location.building), location) @staticmethod def validate_website(url): """Validate url by sending HEAD request and analyzing response. Args: url (str): URL to validate. Raises: JsonValidationError: If URL is invalid. """ c = httplib.HTTPConnection(url) c.request('HEAD', '') # NOTE: 200 - good status # 301 - redirected if c.getresponse().status == 200 or c.getresponse().status == 301: return raise JsonValidationError('invalid website w/url "%s"'.format(url), {'url': url}) def validate_time_range(self, start, end): """Validate start time is less than end time. There exists an unhandled case if the end time is midnight. Args: start (str): Start time. end (str): End time. Raises: JsonValidationError: If time range invalid. """ try: start, end = map(dparser.parse, [start, end]) except ValueError: raise JsonValidationError('invalid time range {}-{}', start, end) if start >= end: raise JsonValidationError('start {} >= end {}', start, end) # NOTE: there exists an unhandled case if the end time is midnight. def validate_directory(self, directory): """Validate directory. Args: directory (str, dict): Directory to validate. May be either path or object. Raises: e: TODO """ if isinstance(directory, str): try: name = directory directory = dir_to_dict(directory) directory['name'] = name except IOError as e: print('ERROR: invalid directory path\n' + str(e), file=sys.stderr) raise e Validator.schema_validate(directory, *Validator.SCHEMAS.directory)