def start_requests(self):
while True:
products = Product.select().where(Product.review_last_checked == None).limit(120)
if len(products) == 0:
break
work_queue = Queue()
processes = []
for product in products:
work_queue.put(product)
for w in range(self.process_num):
db = init_database()
browser = get_browser(profile=None, disable_js=True, disable_image=True, headless=True)
p = Process(target=self.worker, args=(work_queue, browser, db))
p.start()
processes.append([p, browser, db])
for p in processes:
try:
p[0].join()
except (KeyboardInterrupt, SystemExit):
print('Exiting...Please wait!')
p[0].terminate()
p[0].join()
p[2].close()
def start_requests(self):
while True:
# products = Product.select().where(Product.name == None).limit(6)
sql = "select asin from products where name is null limit 120"
cursor = init_database().execute_sql(sql)
rows = cursor.fetchall()
if len(rows) == 0:
print('No more products to process...')
break
work_queue = Queue()
processes = []
for row in rows:
work_queue.put(row[0])
for w in range(self.process_num):
db = init_database()
browser = get_browser(profile=None,
disable_js=True,
disable_image=True,
headless=True)
p = Process(target=self.worker, args=(work_queue, browser, db))
p.start()
processes.append([p, browser, db])
for p in processes:
try:
p[0].join()
except (KeyboardInterrupt, SystemExit):
print('Exiting...Please wait!')
p[0].terminate()
p[0].join()
p[2].close()
def get_browser():
#, disable_js=True, disable_image=True, headless=True
return get_browser(profile=None)
def get_browser():
return get_browser(profile=None)