def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
filename='banners.log',
filemode='w')
parser = argparse.ArgumentParser(
description='Retrieves the Banners of the given store')
parser.add_argument('store',
type=str,
help='The name of the store to be parsed')
parser.add_argument('--extra_args',
type=json.loads,
nargs='?',
default={},
help='Optional arguments to pass to the parser '
'(usually username/password) for private sites)')
args = parser.parse_args()
store = get_store_class_by_name(args.store)
banners = store.banners(extra_args=args.extra_args)
if banners:
for banner in banners:
if "picture" in banner:
banner.pop('picture')
print(json.dumps(banner, indent=4), '\n')
else:
print('No banners found')
def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
filename='products_for_url.log',
filemode='w')
parser = argparse.ArgumentParser(
description='Retrieves the products of the given store / url')
parser.add_argument('store',
type=str,
help='The name of the store to be parsed')
parser.add_argument('url', type=str, help='Discovery URL of the products')
parser.add_argument('--extra_args',
type=json.loads,
nargs='?',
default={},
help='Optional arguments to pass to the parser '
'(usually username/password) for private sites)')
args = parser.parse_args()
store = get_store_class_by_name(args.store)
products = store.products_for_url(url=args.url,
category='Unknown',
extra_args=args.extra_args)
if products:
for product in products:
print(product, '\n')
else:
print('No products found')
def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
filename='discover_urls_for_keyword.log',
filemode='w')
parser = argparse.ArgumentParser(
description='Discovers the URLs of the given store and keyword')
parser.add_argument('store', type=str,
help='The name of the store to be parsed')
parser.add_argument('keyword', type=str, nargs='*',
help='Specific keyword to be searched')
parser.add_argument('threshold', type=int,
help='The amount of urls to retrieve')
parser.add_argument('--extra_args', type=json.loads, nargs='?', default={},
help='Optional arguments to pass to the parser '
'(usually username/password) for private sites)')
args = parser.parse_args()
store = get_store_class_by_name(args.store)
keyword = args.keyword[0]
threshold = args.threshold
results = store.discover_urls_for_keyword(
keyword, threshold,
extra_args=args.extra_args)
for url in results:
print('\n{}'.format(url))
print('Total: {} URLs'.format(len(results)))
def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
filename='products.log',
filemode='w')
parser = argparse.ArgumentParser(
description='Discovers the URLs of the given store and keyword')
parser.add_argument('store', type=str,
help='The name of the store to be parsed')
parser.add_argument('keyword', type=str, nargs='*',
help='Specific keyword to be searched')
parser.add_argument('threshold', type=int,
help='The amount of urls to retrieve')
parser.add_argument('--with_async', type=bool, nargs='?', default=False,
const=True,
help='Use async tasks (celery)')
parser.add_argument('--extra_args', type=json.loads, nargs='?', default={},
help='Optional arguments to pass to the parser '
'(usually username/password) for private sites)')
args = parser.parse_args()
store = get_store_class_by_name(args.store)
keyword = args.keyword[0]
threshold = args.threshold
available_products = 0
unavailable_products = 0
products_data = store.products_for_keyword(
keyword, threshold,
use_async=args.with_async,
extra_args=args.extra_args)
for product in products_data['products']:
if product.is_available():
available_products += 1
else:
unavailable_products += 1
print(product, '\n')
urls_with_error = products_data['discovery_urls_without_products']
print('Discovery URLs without products:')
if not urls_with_error:
print('* No empty URLs found')
for url_with_error in urls_with_error:
print('* {}'.format(url_with_error))
print()
print('Available: {}'.format(available_products))
print('Unavailable: {}'.format(unavailable_products))
print('With error: {}'.format(len(urls_with_error)))
print('Total: {}'.format(available_products + unavailable_products +
len(urls_with_error)))
def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
filename='discover_urls_for_categories.log',
filemode='w')
parser = argparse.ArgumentParser(
description='Discovers the URLs of the given store and (optional) '
'categories')
parser.add_argument('store',
type=str,
help='The name of the store to be parsed')
parser.add_argument('--categories',
type=str,
nargs='*',
help='Specific categories to be parsed')
parser.add_argument('--with_async',
type=bool,
nargs='?',
default=False,
const=True,
help='Use asynchronous tasks (celery)')
parser.add_argument('--extra_args',
type=json.loads,
nargs='?',
default={},
help='Optional arguments to pass to the parser '
'(usually username/password) for private sites)')
args = parser.parse_args()
store = get_store_class_by_name(args.store)
result = store.discover_entries_for_categories(categories=args.categories,
use_async=args.with_async,
extra_args=args.extra_args)
for url, entry_metadata in result.items():
print('\n{} ({} - {})'.format(url, entry_metadata['category'],
entry_metadata['category_weight']))
positions = entry_metadata['positions']
if positions:
for section_name, value in positions.items():
print('* {} in {}'.format(value, section_name))
else:
print('No positions given')
print('Total: {} URLs'.format(len(result)))
class WtbBrand(models.Model):
name = models.CharField(max_length=100)
prefered_brand = models.CharField(max_length=100, blank=True, null=True)
storescraper_class = models.CharField(max_length=100, blank=True,
null=True)
website = models.ForeignKey(Website, on_delete=models.CASCADE)
stores = models.ManyToManyField(Store)
objects = WtbBrandQuerySet.as_manager()
scraper = property(
lambda self: get_store_class_by_name(self.storescraper_class))
def __str__(self):
return self.name
def update_entities(self,
discover_urls_concurrency=None,
products_for_url_concurrency=None,
use_async=None, update_log=None):
assert self.storescraper_class
scraper = self.scraper
if update_log:
update_log.status = update_log.IN_PROCESS
update_log.save()
# First pass of product retrieval
def log_update_error(exception):
if update_log:
update_log.status = update_log.ERROR
desired_filename = 'logs/scrapings/{}_{}.json'.format(
self, timezone.localtime(
update_log.creation_date).strftime('%Y-%m-%d_%X'))
storage = PrivateS3Boto3Storage()
real_filename = storage.save(
desired_filename, ContentFile(
str(exception).encode('utf-8')))
update_log.registry_file = real_filename
update_log.save()
try:
scraped_products_data = scraper.products(
discover_urls_concurrency=discover_urls_concurrency,
products_for_url_concurrency=products_for_url_concurrency,
use_async=use_async
)
except Exception as e:
log_update_error(e)
raise
# self.update_with_scraped_products(scraped_products_data['products'],
# update_log=update_log)
scraped_products = scraped_products_data['products']
scraped_products_dict = iterable_to_dict(scraped_products, 'key')
entities_to_be_updated = self.wtbentity_set.select_related()
categories_dict = iterable_to_dict(Category, 'storescraper_name')
for entity in entities_to_be_updated:
scraped_product_for_update = scraped_products_dict.pop(
entity.key, None)
entity.update_with_scraped_product(
scraped_product_for_update)
for scraped_product in scraped_products_dict.values():
WtbEntity.create_from_scraped_product(
scraped_product,
self,
categories_dict[scraped_product.category]
)
if update_log:
update_log.status = update_log.SUCCESS
serialized_scraping_info = [p.serialize()
for p in scraped_products]
storage = PrivateS3Boto3Storage()
scraping_record_file = ContentFile(json.dumps(
serialized_scraping_info, indent=4).encode('utf-8'))
desired_filename = 'logs/scrapings/{}_{}.json'.format(
self, timezone.localtime(update_log.creation_date).strftime(
'%Y-%m-%d_%X'))
real_filename = storage.save(desired_filename,
scraping_record_file)
update_log.registry_file = real_filename
update_log.save()
class Meta:
ordering = ('name', )
permissions = [
('view_wtb_brand', 'Can view the WTB brand'),
('is_wtb_brand_staff', 'Is staff of this WTB brand'),
('backend_view_wtb', 'Display the WTB menu in the backend'),
]
class Store(models.Model):
name = models.CharField(max_length=255, db_index=True, unique=True)
country = models.ForeignKey(Country, on_delete=models.CASCADE)
is_active = models.BooleanField(default=True)
storescraper_class = models.CharField(max_length=255, db_index=True)
storescraper_extra_args = models.CharField(max_length=255,
null=True,
blank=True)
type = models.ForeignKey(StoreType, on_delete=models.CASCADE)
logo = ImageField(upload_to='store_logos')
objects = StoreQuerySet.as_manager()
scraper = property(
lambda self: get_store_class_by_name(self.storescraper_class))
def __str__(self):
return self.name
def update_pricing(self,
categories=None,
discover_urls_concurrency=None,
products_for_url_concurrency=None,
use_async=None,
update_log=None):
assert self.is_active
scraper = self.scraper
categories = self.sanitize_categories_for_update(categories)
if not categories:
return
if update_log:
update_log.status = update_log.IN_PROCESS
update_log.save()
extra_args = {}
if self.storescraper_extra_args:
extra_args = json.loads(self.storescraper_extra_args)
# First pass of product retrieval
def log_update_error(exception):
if update_log:
update_log.status = update_log.ERROR
desired_filename = 'logs/scrapings/{}_{}.json'.format(
self,
timezone.localtime(
update_log.creation_date).strftime('%Y-%m-%d_%X'))
storage = PrivateS3Boto3Storage()
real_filename = storage.save(
desired_filename,
ContentFile(str(exception).encode('utf-8')))
update_log.registry_file = real_filename
update_log.save()
try:
scraped_products_data = scraper.products(
categories=[c.storescraper_name for c in categories],
extra_args=extra_args,
discover_urls_concurrency=discover_urls_concurrency,
products_for_url_concurrency=products_for_url_concurrency,
use_async=use_async)
except Exception as e:
log_update_error(e)
raise
# Second pass of product retrieval, for the products mis-catalogued
# in the store
extra_entities = self.entity_set.filter(
category__in=categories
).exclude(scraped_category__in=categories, ).exclude(
# Exclude the entities that we already scraped previously. This
# happens when categories is None and is sanitized to include
# the categories of these entities.
key__in=[e.key for e in scraped_products_data['products']])
extra_entities_args = [
dict([('url', e.discovery_url),
('category', e.scraped_category.storescraper_name)])
for e in extra_entities
]
extra_products_task_signature = scraper.products_for_urls_task.s(
self.storescraper_class,
extra_entities_args,
extra_args=extra_args,
products_for_url_concurrency=products_for_url_concurrency,
use_async=use_async)
extra_products_task_signature.set(queue='storescraper')
# Prevents Celery error for running a task inside another
with allow_join_result():
try:
extra_products_data = \
extra_products_task_signature.delay().get()
except Exception as e:
log_update_error(e)
raise
scraped_products = scraped_products_data['products'] + \
extra_products_data['products']
discovery_urls_without_products = \
scraped_products_data['discovery_urls_without_products'] + \
extra_products_data['discovery_urls_without_products']
self.update_with_scraped_products(categories,
scraped_products,
discovery_urls_without_products,
update_log=update_log)
def update_pricing_from_json(self, json_data, update_log=None):
assert self.is_active
categories = Category.objects.filter(
storescraper_name__in=json_data['categories'])
products = [
StorescraperProduct.deserialize(product)
for product in json_data['products']
]
self.update_with_scraped_products(
categories,
products,
json_data['discovery_urls_without_products'],
update_log=update_log)
def update_with_scraped_products(self,
categories,
scraped_products,
discovery_urls_without_products,
update_log=None):
from solotodo.models import Currency, Entity
assert self.is_active
scraped_products_dict = iterable_to_dict(scraped_products, 'key')
entities_to_be_updated = self.entity_set.filter(
Q(category__in=categories)
| Q(key__in=scraped_products_dict.keys())).select_related()
categories_dict = iterable_to_dict(Category, 'storescraper_name')
currencies_dict = iterable_to_dict(Currency, 'iso_code')
for entity in entities_to_be_updated:
scraped_product_for_update = scraped_products_dict.pop(
entity.key, None)
if scraped_product_for_update:
category = categories_dict[scraped_product_for_update.category]
currency = currencies_dict[scraped_product_for_update.currency]
else:
category = None
currency = None
entity.update_with_scraped_product(scraped_product_for_update,
category, currency)
for scraped_product in scraped_products_dict.values():
Entity.create_from_scraped_product(
scraped_product,
self,
categories_dict[scraped_product.category],
currencies_dict[scraped_product.currency],
)
if update_log:
update_log.status = update_log.SUCCESS
update_log.available_products_count = len(
list(filter(lambda x: x.is_available(), scraped_products)))
update_log.unavailable_products_count = len(
list(filter(lambda x: not x.is_available(), scraped_products)))
update_log.discovery_urls_without_products_count = len(
discovery_urls_without_products)
serialized_scraping_info = {
'categories': [c.storescraper_name for c in categories],
'discovery_urls_without_products':
discovery_urls_without_products,
'products': [p.serialize() for p in scraped_products]
}
storage = PrivateS3Boto3Storage()
scraping_record_file = ContentFile(
json.dumps(serialized_scraping_info, indent=4).encode('utf-8'))
desired_filename = 'logs/scrapings/{}_{}.json'.format(
self,
timezone.localtime(
update_log.creation_date).strftime('%Y-%m-%d_%X'))
real_filename = storage.save(desired_filename,
scraping_record_file)
update_log.registry_file = real_filename
update_log.save()
def scraper_categories(self):
return Category.objects.filter(
storescraper_name__in=self.scraper.categories())
def sanitize_categories_for_update(self, original_categories=None):
sanitized_categories = self.scraper_categories()
if original_categories:
sanitized_categories &= original_categories
# If we have entities whose categories differ between our backend
# and the store itself add their categories manually to the list of
# categories to be updated if the original categories are not
# given
# Example:
# 1. "Sanitize all the categories from AbcDin"
# (original_categories = None)
# 2. Load the default categories from AbcDin [Television, etc...]
# 3. We may have an entity from AbcDin with mismatched category.
# For example they had a Processor in the Notebook section
# Normally this entity would never be updated as Processor is not
# part of AbcDin scraper, so add "Processor" to the list of categories
if original_categories is None:
extra_category_ids = self.entity_set.exclude(
category__in=sanitized_categories).values('category')
extra_categories = Category.objects.filter(
pk__in=[e['category'] for e in extra_category_ids])
sanitized_categories |= extra_categories
return sanitized_categories
class Meta:
app_label = 'solotodo'
ordering = ['name']
permissions = (
['view_store', 'Can view the store'],
['view_store_update_logs', 'Can view the store update logs'],
['view_store_stocks', 'Can view the store entities stock'],
['update_store_pricing', 'Can update the store pricing'],
['is_store_staff', 'Is staff of the store'],
('view_store_leads', 'View the leads associated to this store'),
# "Backend" permissions are used exclusively for UI purposes, they
# are not used at the API level
['backend_list_stores', 'Can view store list in backend'],
)