def __init__(self,
school,
data=None,
output=None,
diff=True,
load=True,
display_progress_bar=True,
tracker=NullTracker()):
with open(data, 'r') as f:
data = json.load(f)
self.meta = data.pop('$meta')
self.data = [DotDict(obj) for obj in data.pop('$data')]
self.cache = DotDict(dict(course={'code': '_'}, section={'code': '_'}))
self.school = school
self.adapter = DigestionAdapter(school, self.cache)
self.strategy = self._resolve_strategy(diff, load, output)
# Setup tracker for digestion and progress bar.
self.tracker = tracker
self.tracker.mode = 'digesting'
if display_progress_bar:
self.tracker.add_viewer(ProgressBar('{total}'))
def __init__(self, school, meta, tracker=NullTracker()):
"""Construct Digestor instance.
Args:
school (str): Description
data (None, optional): Description
output (None, optional): Description
diff (bool, optional): Description
load (bool, optional): Description
tracker (TYPE, optional): Description
"""
# with open(data, 'r') as f:
# data = json.load(f)
self.cache = DotDict(
dict(course={"code": None}, section={"code": None}))
self.school = school
self.adapter = DigestionAdapter(
school, self.cache,
SCHOOLS_MAP[self.school].short_course_weeks_limit)
self.meta = meta
# Setup tracker for digestion and progress bar.
self.tracker = tracker
self.tracker.mode = "digesting"
def __init__(self,
school,
config_path,
output_path,
output_error_path,
break_on_error=True,
break_on_warning=False,
display_progress_bar=True,
skip_duplicates=True,
validate=True,
tracker=NullTracker()):
"""Construct ingestor object and resolve options.
Args:
school (string): The school code (e.g. jhu, gw, umich).
config_path (str): Configuration file path.
output_path (str): Output path.
output_error_path (str): Error output path.
break_on_error (bool, optional): Stop ingesting on error.
break_on_warning (bool, optional): Stop ingesting on warning.
display_progress_bar (bool, optional): display progress bar
skip_duplicates (bool, optional): Skip ingesting courses
that have already been seen.
validate (bool, optional): Perform validation.
tracker (library.tracker, optional): tracker object
"""
self.school = school
self.validate = validate
self.break_on_error = break_on_error
self.break_on_warning = break_on_warning
self.skip_duplicates = skip_duplicates
self.tracker = tracker
self.hoarder = Hoarder()
self.tracker.add_viewer(self.hoarder)
self.tracker.school = school
# Initialize loggers for json and errors.
self.json = JSONStreamWriter(output_path, type_=dict).enter()
self.data_list = self.json.write('$data', type_=list).enter()
self.logger = Logger(errorfile=output_error_path)
if self.validate:
self.validator = Validator(config_path, tracker=self.tracker)
# Inherit dictionary functionality.
super(Ingestor, self).__init__()