def open_spider(self, spider):
client = HubstorageClient(auth=settings.SHUB_KEY)
project = client.get_project(settings.SHUB_PROJ_ID)
self.data_stores = {}
for product_name in get_product_names():
self.data_stores[product_name] = project.collections.new_store(
product_name)
def main(args):
items = []
for prod_name in get_product_names():
fetcher = DealsFetcher(prod_name, args.apikey, args.project, args.days * 24)
checker = DealsChecker(*fetcher.get_deals(), args.threshold)
best_deal = checker.get_best_deal()
if checker.is_from_latest_crawl(best_deal):
items.append(best_deal)
if items:
send_email_alert(items)
def open_spider(self, spider):
self.filename = "data/" + datetime.now().strftime("%Y%m%d") + ".json"
if not os.path.exists(os.path.dirname(self.filename)):
try:
os.makedirs(os.path.dirname(self.filename))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
self.data_stores = {}
try:
with open(self.filename, 'r') as fp:
self.data_stores = json.load(fp)
except:
pass
for product_name in get_product_names():
if product_name not in self.data_stores:
self.data_stores[product_name] = {}
def open_spider(self, spider):
client = HubstorageClient(auth=settings.021fda8005214eac944950e2e96ffc92)
project = client.get_project(settings.301640)
self.data_stores = {}
for product_name in get_product_names():
self.data_stores[product_name] = project.collections.new_store(product_name)