def handle(self, *args, **options):
"""Logic of the command.
Args:
*args: Args of command.
**options: Command options.
"""
tracker = Tracker()
tracker.mode = 'ingesting'
self.stat_view = StatView()
tracker.add_viewer(self.stat_view)
if options['display_progress_bar']:
tracker.add_viewer(
StatProgressBar('{valid}/{total}', statistics=self.stat_view))
tracker.start()
for parser_type in options['types']:
for school in options['schools']:
tracker.school = school
try:
parsing = __import__(
'parsing.schools.{school}.{parser_type}'.format(
school=school, parser_type=parser_type))
parser = eval(
'parsing.schools.{school}.{parser_type}.Parser'.format(
school=school, parser_type=parser_type))
self.run(parser, tracker, options, parser_type, school)
except ImportError:
logging.exception('Invalid parser')
continue
tracker.end()
def run(self, tracker, school, data_type, options):
"""Run the command."""
tracker.school = school
tracker.mode = "validating"
if options["display_progress_bar"]:
tracker.add_viewer(
StatProgressBar("{valid}/{total}", statistics=self.stat_view),
name="progressbar",
)
logger = logging.getLogger("parsing.schools." + school)
logger.debug("Digest command options:" + str(options))
# Load config file to dictionary.
if isinstance(options["config"], str):
with open(options["config"].format(school=school, type=data_type),
"r") as file:
options["config"] = json.load(file)
try:
Validator(options["config"],
tracker=tracker).validate_self_contained(
options["data"].format(school=school,
type=data_type),
break_on_error=True,
break_on_warning=options.get("break_on_warning"),
display_progress_bar=options["display_progress_bar"],
)
except (ValidationError, ValidationWarning, Exception):
logging.exception("Failed validation before digestion")
return # Skip digestion for this school.
if options["display_progress_bar"]:
tracker.remove_viewer("progressbar")
tracker.add_viewer(ETAProgressBar(), name="progressbar")
tracker.mode = "digesting"
with open(options["data"].format(school=school, type=data_type),
"r") as file:
data = json.load(file)
try:
Digestor(school, meta=data["$meta"], tracker=tracker).digest(
data["$data"],
diff=options["diff"],
load=options["load"],
output=options["output_diff"].format(school=school,
type=data_type),
)
except DigestionError:
logging.exception("Failed digestion")
except PipelineException:
logging.expection("Failed digestion w/in pipeline")
except Exception:
logging.exception("Failed digestion with uncaught exception")
logging.info("Digestion overview for " + school + ": " +
str(self.stat_view.report()))
def run(self, tracker, school, data_type, options):
"""Run the command."""
tracker.school = school
tracker.mode = 'validating'
if options['display_progress_bar']:
tracker.add_viewer(StatProgressBar('{valid}/{total}',
statistics=self.stat_view),
name='progressbar')
logger = logging.getLogger('parsing.schools.' + school)
logger.debug('Digest command options:' + str(options))
# Load config file to dictionary.
if isinstance(options['config'], str):
with open(options['config'].format(school=school, type=data_type),
'r') as file:
options['config'] = json.load(file)
try:
Validator(options['config'],
tracker=tracker).validate_self_contained(
options['data'].format(school=school,
type=data_type),
break_on_error=True,
break_on_warning=options.get('break_on_warning'),
display_progress_bar=options['display_progress_bar'])
except (ValidationError, ValidationWarning, Exception):
logging.exception('Failed validation before digestion')
return # Skip digestion for this school.
if options['display_progress_bar']:
tracker.remove_viewer('progressbar')
tracker.add_viewer(ETAProgressBar(), name='progressbar')
tracker.mode = 'digesting'
with open(options['data'].format(school=school, type=data_type),
'r') as file:
data = json.load(file)
try:
Digestor(school, meta=data['$meta'], tracker=tracker).digest(
data['$data'],
diff=options['diff'],
load=options['load'],
output=options['output_diff'].format(school=school,
type=data_type))
except DigestionError:
logging.exception('Failed digestion')
except PipelineException:
logging.expection('Failed digestion w/in pipeline')
except Exception:
logging.exception('Failed digestion with uncaught exception')
logging.info('Digestion overview for ' + school + ': ' +
str(self.stat_view.report()))
def handle(self, *args, **options):
"""Logic of the command.
Args:
*args: Args of command.
**options: Command options.
"""
tracker = Tracker()
tracker.mode = 'validating'
if options['display_progress_bar']:
tracker.add_viewer(StatProgressBar('{valid}/{total}'))
tracker.start()
for parser_type in options['types']:
for school in options['schools']:
self.run(options, school, parser_type, tracker)
def handle(self, *args, **options):
"""Logic of the command.
Args:
*args: Args of command.
**options: Command options.
"""
tracker = Tracker()
tracker.mode = 'ingesting'
self.stat_view = StatView()
tracker.add_viewer(self.stat_view)
if options['display_progress_bar']:
tracker.add_viewer(
StatProgressBar('{valid}/{total}', statistics=self.stat_view))
tracker.start()
for parser_type in options['types']:
for school in options['schools']:
tracker.school = school
self.run(SCHOOLS_MAP[school].parsers[parser_type], tracker,
options, parser_type, school)
tracker.end()