def scrape(self, mapping=None, keys: List[str] = None, exclude: List[str] = None) -> None:
"""
Scrape data at self.url and parse into attributes
Parameters
----------
keys : List[str]
Specify what keys to exclusively load as strings in a list. If no
keys are specified then the scrape will default load all data points.
exclude : List[str]
Specify what keys to exclude from being loaded. If no keys are
specified, then no data points will be excluded.
"""
if mapping is None:
mapping = self._Mapping
if keys is None:
keys = []
if exclude is None:
exclude = []
# If the passed source was already an object, construct data from
# source else parse it
if type(self.source) is type(self):
scraped_dict = self.source.to_dict()
else:
self.json_dict = self._get_json_from_source(self.source)
self.flat_json_dict = FlatJSONDict(self.json_dict)
scraped_dict = parse_json_from_mapping(
json_dict=self.flat_json_dict,
map_dict=self._Mapping.return_mapping(keys=keys, exclude=exclude),
)
self._load_into_namespace(scraped_dict)
def scrape(self, mapping=None, keys: List[str] = None, exclude: List[str] = None) -> None:
"""
Scrape data from self.source and load as instance attributes
Parameters
----------
mapping : Dict[str, deque]
Dictionary of parsing queue's that tell the JSON engine how to
process the JSON data
keys : List[str]
List of strings that correspond to desired attributes for scraping
exclude : List[str]
List of strings that correspond to which attributes to exclude from
being scraped
"""
if mapping is None:
mapping = self._Mapping
if keys is None:
keys = []
if exclude is None:
exclude = []
# If the passed source was already an object, construct data from
# source else parse it
if type(self.source) is type(self):
scraped_dict = self.source.to_dict()
else:
self.json_dict = self._get_json_from_source(self.source)
self.flat_json_dict = FlatJSONDict(self.json_dict)
scraped_dict = parse_json_from_mapping(
json_dict=self.flat_json_dict,
map_dict=self._Mapping.return_mapping(keys=keys, exclude=exclude),
)
self._load_into_namespace(scraped_dict)
def load(self, keys: List[str] = [], exclude: List[str] = []) -> None:
"""
Scrape data at self.url and parse into attributes
Parameters
----------
keys : List[str]
Specify what keys to exclusively load as strings in a list. If no
keys are specified then the scrape will default load all data points.
exclude : List[str]
Specify what keys to exclude from being loaded. If no keys are
specified, then no data points will be excluded.
"""
if type(keys) == str:
keys = [keys]
if type(exclude) == str:
exclude = [exclude]
self.json_dict = json_from_url(self.url)
self.flat_json_dict = FlatJSONDict(self.json_dict)
scraped_dict = parse_json_from_mapping(
json_dict=self.flat_json_dict,
map_dict=self._Mapping.return_mapping(keys=keys, exclude=exclude),
)
self._load_into_namespace(scraped_dict=scraped_dict)
self.scrape_timestamp = datetime.datetime.now()