class BaseParser:
"""Abstract base parser for data pipeline parsers.
Attributes:
extractor (:obj:`parsing.library.extractor.Extractor`)
ingestor (:obj:`parsing.library.ingestor.Ingestor`)
requester (:obj:`parsing.library.requester.Requester`)
school (:obj:`str`): School that parser is for.
"""
__metaclass__ = ABCMeta
def __init__(self,
school,
config=None,
output_path=None,
output_error_path=None,
break_on_error=True,
break_on_warning=False,
skip_duplicates=True,
display_progress_bar=False,
validate=True,
tracker=None):
"""Create base parser instance.
Args:
school (str): The subject school.
config (None, dict, optional): Configuration dictionary.
output_path (None, str, optional)
output_error_path (None, str, optional)
break_on_error (bool, optional)
break_on_warning (bool, optional)
skip_duplicates (bool, optional): TODO
display_progress_bar (bool, optional)
validate (bool, optional): Flag to validate or not.
tracker (None, Tracker, optional): Tracker to track parse.
"""
self.school = school
self.requester = Requester()
self.tracker = tracker
self.ingestor = Ingestor(config,
output_path,
break_on_error=break_on_error,
break_on_warning=break_on_warning,
skip_duplicates=skip_duplicates,
display_progress_bar=display_progress_bar,
validate=validate,
tracker=self.tracker)
@abstractmethod
def start(self, **kwargs):
"""Start the parse.
Args:
**kwargs: expanded in child parser.
"""
def end(self):
"""Finish the parse."""
self.ingestor.end()
class BaseParser:
"""Abstract base parser for data pipeline parsers.
Attributes:
extractor (:obj:`parsing.library.extractor.Extractor`)
ingestor (:obj:`parsing.library.ingestor.Ingestor`)
requester (:obj:`parsing.library.requester.Requester`)
school (:obj:`str`): School that parser is for.
"""
__metaclass__ = ABCMeta
def __init__(self, school,
config=None,
output_path=None,
output_error_path=None,
break_on_error=True,
break_on_warning=False,
skip_duplicates=True,
display_progress_bar=False,
validate=True,
tracker=None):
"""Create base parser instance.
Args:
school (str): The subject school.
config (None, dict, optional): Configuration dictionary.
output_path (None, str, optional)
output_error_path (None, str, optional)
break_on_error (bool, optional)
break_on_warning (bool, optional)
skip_duplicates (bool, optional): TODO
display_progress_bar (bool, optional)
validate (bool, optional): Flag to validate or not.
tracker (None, Tracker, optional): Tracker to track parse.
"""
self.school = school
self.requester = Requester()
self.tracker = tracker
self.ingestor = Ingestor(config,
output_path,
break_on_error=break_on_error,
break_on_warning=break_on_warning,
skip_duplicates=skip_duplicates,
display_progress_bar=display_progress_bar,
validate=validate,
tracker=self.tracker)
@abstractmethod
def start(self, **kwargs):
"""Start the parse.
Args:
**kwargs: expanded in child parser.
"""
def end(self):
"""Finish the parse."""
self.ingestor.end()
def __init__(self, school,
config=None,
output_path=None,
output_error_path=None,
break_on_error=True,
break_on_warning=False,
skip_duplicates=True,
display_progress_bar=False,
validate=True,
tracker=None):
"""Create base parser instance.
Args:
school (str): The subject school.
config (None, dict, optional): Configuration dictionary.
output_path (None, str, optional)
output_error_path (None, str, optional)
break_on_error (bool, optional)
break_on_warning (bool, optional)
skip_duplicates (bool, optional): TODO
display_progress_bar (bool, optional)
validate (bool, optional): Flag to validate or not.
tracker (None, Tracker, optional): Tracker to track parse.
"""
self.school = school
self.requester = Requester()
self.tracker = tracker
self.ingestor = Ingestor(config,
output_path,
break_on_error=break_on_error,
break_on_warning=break_on_warning,
skip_duplicates=skip_duplicates,
display_progress_bar=display_progress_bar,
validate=validate,
tracker=self.tracker)
def __init__(
self,
school,
config=None,
output_path=None,
output_error_path=None,
break_on_error=True,
break_on_warning=False,
skip_duplicates=True,
display_progress_bar=False,
validate=True,
tracker=None,
):
"""Create base parser instance.
Args:
school (str): The subject school.
config (None, dict, optional): Configuration dictionary.
output_path (None, str, optional)
output_error_path (None, str, optional)
break_on_error (bool, optional)
break_on_warning (bool, optional)
skip_duplicates (bool, optional): TODO
display_progress_bar (bool, optional)
validate (bool, optional): Flag to validate or not.
tracker (None, Tracker, optional): Tracker to track parse.
"""
self.school = school
self.requester = Requester()
self.tracker = tracker
self.ingestor = Ingestor(
config,
output_path,
break_on_error=break_on_error,
break_on_warning=break_on_warning,
skip_duplicates=skip_duplicates,
display_progress_bar=display_progress_bar,
validate=validate,
tracker=self.tracker,
)
def __init__(self,
school,
config_path=None,
output_path=None,
output_error_path=None,
break_on_error=True,
break_on_warning=False,
skip_duplicates=True,
display_progress_bar=False,
validate=True,
tracker=None):
"""Create base parser instance.
Args:
school (str): Description
config_path (None, str, optional): Description
output_path (None, str, optional): Description
output_error_path (None, str, optional): Description
break_on_error (bool, optional): Description
break_on_warning (bool, optional): Description
skip_duplicates (bool, optional): Description
display_progress_bar (bool, optional): Description
validate (bool, optional): Description
tracker (None, optional): Description
"""
self.school = school
self.requester = Requester()
self.extractor = Extractor()
self.ingestor = Ingestor(school,
config_path,
output_path,
output_error_path,
break_on_error=break_on_error,
break_on_warning=break_on_warning,
skip_duplicates=skip_duplicates,
display_progress_bar=display_progress_bar,
validate=validate,
tracker=tracker)
class BaseParser:
"""Abstract base parser for data pipeline parsers.
Attributes:
extractor (:obj:`parsing.library.extractor.Extractor`)
ingestor (:obj:`parsing.library.ingestor.Ingestor`)
requester (:obj:`parsing.library.requester.Requester`)
school (:obj:`str`): School that parser is for.
"""
__metaclass__ = ABCMeta
def __init__(self,
school,
config_path=None,
output_path=None,
output_error_path=None,
break_on_error=True,
break_on_warning=False,
skip_duplicates=True,
display_progress_bar=False,
validate=True,
tracker=None):
"""Create base parser instance.
Args:
school (str): Description
config_path (None, str, optional): Description
output_path (None, str, optional): Description
output_error_path (None, str, optional): Description
break_on_error (bool, optional): Description
break_on_warning (bool, optional): Description
skip_duplicates (bool, optional): Description
display_progress_bar (bool, optional): Description
validate (bool, optional): Description
tracker (None, optional): Description
"""
self.school = school
self.requester = Requester()
self.extractor = Extractor()
self.ingestor = Ingestor(school,
config_path,
output_path,
output_error_path,
break_on_error=break_on_error,
break_on_warning=break_on_warning,
skip_duplicates=skip_duplicates,
display_progress_bar=display_progress_bar,
validate=validate,
tracker=tracker)
@abstractmethod
def start(self, **kwargs):
"""Start the parse.
Args:
**kwargs: pass-through to child parser.
"""
def end(self):
"""Finish the parse."""
self.ingestor.end()