def get_offers_by_product_id(self, id, results=1):
# TEMP: fazendo um gato com a api v1
# http://bws.buscape.com.br/service/findOfferList/lomadee/65717751673178504d42633d/BR/?sourceId=36029361&productid=657637&format=json&encode=utf-8&sort=price&results=10
url = "http://bws.buscape.com.br/service/findOfferList/lomadee/{0}/BR/?"
url = url.format("65717751673178504d42633d")
url = url + "{0}&{1}&".format("sourceId=36029361", "productid=" + str(id) )
url = url + "format=json&encode=utf-8&sort=price&results={0}".format( results )
# Pegando as ofertas
json_data = IOUtil.request_json_from_url(url)
# Especificações tecnicas
specification = ""
product = json_data['product'][0]['product']
# print( product )
if 'item' in product['specification'].keys():
for s in product['specification']['item']:
specification += s['item']['label'] + ": " + s['item']['value'][0] + "\n"
names = [ product['productname'] ]
if 'productshortname' in product: names.append( product['productshortname'] )
offers = self._get_offers_from_json(json_data)
for offer in offers:
offer['names'].extend( names )
offer['names'].sort(key=len, reverse=True)
offer['specification'] = specification
return offers
def get_products_id(self, url, category=None, page=None, max_pages=None, return_cat_id=False):
products = []
# pegando uma pagina de ofertas
json_data = IOUtil.request_json_from_url(url.format(page=1 if page is None else page))
products = self._get_product_ids_from_json( json_data, category, return_cat_id )
# retornando somente a pagina solicitada
if page is not None: return products
page = 2
# quantidade de paginas a ser buscadas
total_pages = int(json_data['pagination']['totalPage'])
if max_pages is not None:
total_pages = min(total_pages, max_pages)
# função que busca, processa e retorna todos os produtos de uma pagina
def process_batch(url, page):
json_data = IOUtil.request_json_from_url( url.format(page=page) )
return self._get_product_ids_from_json( json_data, category, return_cat_id )
# Processando todas as paginas paralelamente
nc = mp.cpu_count()
results = Parallel(n_jobs=nc)(delayed(process_batch) (url, i) for i in range(page, total_pages + 1))
# Passando as ofertas para a lista de retorno
for result in results:
products.extend( result )
return products
def get_categories_by_store(page=None, max_pages=None):
categories = {}
# pegando uma pagina de ofertas
json_data = IOUtil.request_json_from_url(url.format(page=1 if page is None else page))
process_json( json_data, categories )
# retornando somente a pagina solicitada
if page is not None: return categories
page = 2
# quantidade de paginas a ser buscadas
total_pages = int(json_data['pagination']['totalPage'])
if max_pages is not None:
total_pages = min(total_pages, max_pages)
for i in tqdm(range(page, total_pages)):
json_data = IOUtil.request_json_from_url( url.format(page=i) )
process_json( json_data, categories )
return categories
def brands_updates(self, _categories=None, to_json=True):
url = self._get_url('category/_id/{0}')
dic_brands = defaultdict(list)
# categories_filter = []
if _categories is None: _categories = categories
# percorre todas as categorias listadas
print("Searching brands by category")
for category in tqdm(_categories):
# buscas as informações da categoria
# print("\tGetting Brands of Category '" + category['name'] + "'")
json_data = IOUtil.request_json_from_url(url.format(category[ 'id' ]))
# pega somente a lista de marcas
json_filters = json_data['categories'][0]['filters']
# Check if has filters
if len(json_filters) == 0: continue
# categories_filter.append( category )
json_brands = json_filters[0]['options']
list_brands = []
# faz uma lista com as marcas e os ids
for brand in json_brands:
list_brands.append(brand)
# salva essa lista em um dict
dic_brands[str(category[ 'id' ])] = list_brands
# print( "Categories without brands: ", count_no_brand)
# gerar um json com todas as marcas
dic_brands = self.remove_brands(dic_brands)
if to_json: IOUtil.dic_to_json('brands.json', None, dic_brands)
# IOUtil.dic_to_json('categories_filter.json', None, categories_filter)
return dic_brands
def process_batch(url, page):
json_data = IOUtil.request_json_from_url( url.format(page=page) )
return self._get_product_ids_from_json( json_data, category, return_cat_id )