class BaseScraper(object):
""" Basic Framework of a simple web communication tool. """
def __init__(self, url):
self.set_url(url)
self.response = None
self.soup = None
self.parser = ParsingManager()
self.history = HistoryManager()
# Getters / Setters/ Loaders
def set_url(self, url):
if type(url) is str:
self.url = parse_url(url) # Returns a EasyUrl() object
else:
self.url = parse_url(str(url))
def get_status_code(self):
""" return self.response.status_code, the status of the last made request. """
if self.response:
return self.response.status_code
else:
return None
def load_soup(self):
""" creates a soup object from self.response. """
if self.response:
self.soup = BS(self.response.text)
# Web Communication Methods
def get(self, payload={}):
self.response = requests.get(self.url, params=payload)
self.load_soup()
self.history.save(self) # Save the state of the passed scraper
def __init__(self, url):
self.set_url(url)
self.response = None
self.soup = None
self.parser = ParsingManager()
self.history = HistoryManager()
