def direct(self):
def browse_each(config):
log.info(Directory.INFO_MAP[0] % (self.market.name, config.name))
try:
map_strs = self.PRODUCT_MAP[config.name]
for map_str in map_strs:
results = self.get_products_prices(map_str)
for product, price in results:
product = Directory.check_product(product)
if not product.id:
Directory.STACK.append((config, product, price))
elif product.part_id:
price.product = product
Directory.set_price(price)
except KeyError:
log.error(Directory.ERROR_MAP[1] % config.name)
cpu = cpu_count()
pool = _ThreadPool(cpu)
for c in self.configs:
pool.apply_async(browse_each, args=(c, ))
pool.close()
pool.join()
def direct(self):
def browse_each(config, url):
product, price = self.get_product_price(url)
if not product and not price:
return
# return self if not exists
product = self.get_product(product)
# set config_id for future re-classify
product.config_id = config.id
if not product.id:
Directory.STACK.append((config, product, price))
else:
price.product = product
Directory.set_price(price)
return
cpu = cpu_count()
pool = _ThreadPool(cpu)
for c, urls in self.config_generator():
for u in urls:
pool.apply_async(browse_each, args=(c, u))
pool.close()
pool.join()