class BasePage(object): __metaclass__ = abc.ABCMeta model = None source = None def __init__(self): self.model = Model() @abc.abstractmethod def scrape(self, input): """Retrieve data from the input source and return a dictionary.""" return def process(self): """Process the scraped data.""" self.soup = BeautifulSoup(self.source) result = self.scrape() market_id = self.model.get_bookmaker_market_id(self.market_type) for competitor, odds in result.iteritems(): competitor_id = None try: """Attempt to get the competitor""" competitor_id = self.model.get_competitor_id(competitor) """Get or create a new bet opportunity""" try: bet_opportunity_id = self.model.get_bet_opportunity_id(\ competitor_id, market_id) except: bet_opportunity_id = self.model.insert_bet_opportunity(\ competitor_id, market_id) odds = self.model.fraction_to_decimal(odds) """ Insert new entries or update the price if it has changed, if so update the timestamp. """ if self.model.get_bet_odds(bet_opportunity_id): if not self.model.get_bet_odds(bet_opportunity_id, odds): self.model.update_bet_odds(bet_opportunity_id, odds) else: self.model.insert_bet_odds(bet_opportunity_id, odds) except: """ The competitors that either do not exist on our side or have different spellings, end up here. """ pass