def send_notificaiton(self, msg: Text, identifier: Text) -> NoReturn:
if self.notification_history.has_been_notified(identifier):
log.info('Already sent notification for identifier %s', identifier)
return
for agent in self.notificaiton_agents:
log.info('Sending notification to %s', agent.name)
log.debug(msg)
try:
agent.send(msg)
self.notification_history.add_history(identifier)
except Exception as e:
log.exception('Failed to send notification', exc_info=True)
def check_search_pages(self) -> List[ProductInfo]:
all_results = []
for page in self.search_pages:
log.info('Checking search page: %s', page)
page_source = self._load_page(page)
if not page_source:
log.error('Did not get page source. Skipping %s', page)
continue
page = BeautifulSoup(page_source, 'html.parser')
all_results += self.parse_search_page(page)
for r in all_results:
log.debug(r)
return all_results
def _load_page(self, url: Text, user_agent=None) -> Optional[Text]:
ua = UserAgent(cache=False)
try:
headers = {'User-Agent': user_agent or ua.chrome}
log.debug('User Agent: %s', headers['User-Agent'])
r = requests.get(url, headers=headers)
except (ConnectionError, Timeout):
log.error('Failed to load URL: %s', url)
return
if r.status_code != 200:
log.error('Unexpected Status Code %s for URL %s', r.status_code,
url)
return
return r.text
def _is_in_stock_search_result(self, search_result: Tag) -> bool:
btn_box = search_result.find('div', {'class': 'item-button-area'})
if not btn_box:
log.error('Did not find button box')
return False
btn = btn_box.find('button')
if not btn:
log.error('Did not find button')
return False
log.debug('Button Text: %s', btn.text)
if btn.text.lower().strip() == 'add to cart':
return True
else:
return False
def _get_price_from_product_page(self,
page: BeautifulSoup) -> Optional[Text]:
buy_box = page.find('div', {'class': 'product-buy-box'})
if not buy_box:
log.error('Failed to find buy box')
return
price_box = buy_box.find('li', {'class': 'price-current'})
if not price_box:
log.error('Failed to find price box')
return
price_text_box = price_box.find('strong')
if not price_text_box:
log.error('Failed to find price text box')
return
price = price_text_box.text
log.debug('Price is %s', price)
return price
def _get_sku_from_product_page(self,
page: BeautifulSoup) -> Optional[Text]:
breadcrumb_box = page.find('ol', {'class': 'breadcrumb'})
if not breadcrumb_box:
log.error('Failed to get breadcrumb box from page')
return
sku_box = breadcrumb_box.find('li', {'class': 'is-current'})
if not sku_box:
log.error('Failed to find SKU box')
return
sku_text_box = sku_box.find('em')
if not sku_text_box:
log.error('Failed to find SKU text box')
return
sku = sku_text_box.text
log.debug('Found SKU %s', sku)
return sku
def is_ignored(self, data: ProductInfo) -> bool:
for kw in self.ignore_title_keywords:
if kw.lower() in data.title.lower():
log.debug('Ignore keyword %s | %s', kw, data.title)
return True
if data.url in self.ignore_urls:
log.debug('Product URL in ignore list')
return True
if data.url in self.notification_sent_urls:
log.debug('Already sent notification for this product')
return True
return False
def _is_combo_page(self, page: BeautifulSoup) -> bool:
"""
Check if a given page is a combo product page. The layout is completely different so parsing will fail
:rtype: bool
:param page: BeatufilSoup Object
"""
breadcrumbs = page.find('ol', {'class': 'breadcrumb'})
if not breadcrumbs:
log.debug('Failed to find breadcrumbs, assuming combo page')
return True
current_item = breadcrumbs.find('li', {'class': 'is-current'})
if not current_item:
log.debug(
'Failed to current item in breadcrumbs, assuming combo page')
return True
if 'combo' in current_item.text.lower():
log.debug('Current page is combo page')
return True
return False
def add_history(self, identifier: Text):
with open(self.history_file_name, 'a') as f:
f.write(identifier + '\n')
log.debug(
'Added 1 entry to notification history with identifer %s',
identifier)