def __init__(self, url, bs_parser="lxml"):
self.parser = WebParser(url)
self.bs_parser = bs_parser
import json
from bot import Bot
from parser import WebParser
import sys
from message import PhoneMessage
def get_amazon_links():
with open('data/links.json') as links_file:
json_links = json.load(links_file)
wipe_products_links = json_links['Wipes']
spray_products_links = json_links['Spray']
return wipe_products_links, spray_products_links
if __name__ == "__main__":
wipe_products_links,spray_products_links = get_amazon_links()
web_parser = WebParser()
bot = Bot(web_parser)
bot.start_scrapying_process(wipe_products_links)
bot.start_scrapying_process(spray_products_links)
stocked_products_urls = bot.webparser.stocked_product_links
#Couldnt find any products that were in stock
if not stocked_products_urls:
sys.exit(1)
else:
phone_message = PhoneMessage(stocked_products_urls)
phone_message.send_message()
def __init__(self, url, timeout=10, checks=3, sleep_interval=0.1):
self.timeout = timeout
self.checks = checks
self.sleep_interval = sleep_interval
self.parser = WebParser(url, rotate_header=True)
def __init__(self, url, timeout=10):
self.timeout = timeout
self.parser = WebParser(url, rotate_header=True)
def test__str__representation(url, expected):
web_parser = WebParser(url)
result = str(web_parser)
assert result == expected
def web_parser():
yield WebParser(TEST_URL)
def __init__(self, rss_feed):
self.parser = WebParser(rss_feed, rotate_header=True)
self.formatter = NewsFormatter()
def __init__(self, keywords_dict, redis_key):
#self.bloom_filter = BloomFilter(redis.StrictRedis(host='localhost', port=6379), 'job_url')
self.parser = WebParser(redis_key)
self.keywords = keywords_dict