async def async_post(session, url, payload, headers={}, cookies={}, printer=None, iteration=None): """ Execute http post request :param session: Session for requests :param url: Url for http requests :param payload: json payload for request :param headers: HTTP Header for request :param cookies: HTTP cookies for request :param printer: (print_function, total, title) for printing :param iteration: [int] list with int for counting print number :return: None """ async with session.post(url, headers=headers, cookies=cookies, json=payload) as response: try: if printer is not None: print_function, total, task = printer iteration[0] += 1 print_function(iteration[0], total, task) if response.status != 201: raise Exception("Error while posting item: " + str(response)) resp = await response.json() return resp except Exception as err: tools.log_error(err) raise err
def process_henry_items(items): """ Processes a list of items from henrys api, converts into pisspricer relevant info :param items: List of items from henrys api :return: List of items for pisspricer api [ { "name": "string", "brand": "string", "categoryId": 0, "subcategoryId": 0, "stdDrinks": 0, "alcoholContent": 0, "volumeEach": 0, "barcode": "string", "salePrice": 0, "price": 0, "stock": "string", "internalSku": "string", "url": "string" } ] """ new_items = [] # Get most item details for henry_item in items: try: new_item = process_henry_item(henry_item) new_items.append(new_item) except Exception as err: tools.log_error(err) print(items[0]) return new_items
def process_stores_page(page_html): """ Processes henrys stores page into list of dictionary store items :param page_html: String of page html :return: List of store dictionary items { name: "required|string", url: "required|string", region: "required|string", region_lat: "numeric", region_lng: "numeric", address: "required|string", postcode: "numeric", lattitude: "numeric", longitude: "numeric" internalId: "string" } """ # Parse the html soup = BeautifulSoup(page_html, features="html.parser") # Get the store info json store_json_list = json.loads(soup.find('store-locations')[':locations']) stores = [] for store_json in store_json_list: try: store = process_store_json(store_json) stores.append(store) except Exception as err: tools.log_error(err) return stores
def handle_message(self, data: str): message = json.loads(data) try: self.visit(message) except NodeNotFoundException as e: log_error( f"Node not found: home ID = {e.home_id}, node ID = {e.node_id}" )
def _post_stores(post_list, regions): """ Iterate through store locations and post to api @param post_list - List of stores from Countdown API to be posted to pisspricer api @param regions - List of regions from pisspricer API """ Countdown.print_progress(0, len(post_list), title=Countdown.name + " Inserting Stores") i = 0 for store_loc in post_list: try: # Get lat and lng address_string = store_loc["name"] + ", " + store_loc["address"] + ", New Zealand" lat, lng, address, postcode, region = tools.geocode_address(address_string) # Set regionId region_id = None for reg in regions: if reg["name"] == region: region_id = reg["regionId"] if region_id is None: new_region_res = requests.post(api.url + "/regions", headers=api.headers, json={"name": region}) if not new_region_res.ok: raise PisspricerApiException(new_region_res, f"posting to /regions {region}") region_id = new_region_res.json()["regionId"] # Set store values store = { "name": store_loc["name"], "url": Countdown.store_url, "brandId": Countdown.cd_brand_id, "lattitude": lat, "longitude": lng, "regionId": region_id, "address": address, "postcode": postcode, "internalId": str(store_loc["id"]) } # Post new store new_store_res = requests.post(api.url + '/stores', headers=api.headers, json=store) if not new_store_res.ok: raise PisspricerApiException(new_store_res, f"posting to /stores {store}") except ApiException as err: tools.log_error(err) finally: i += 1 Countdown.print_progress(i, len(post_list)) if i > 1000: break
async def connect(self) -> bool: try: self.websocket = await websockets.connect(self.uri) asyncio.create_task(self.start_polling()) log_info(f"Connected to {self.uri}") return True except ConnectionRefusedError: log_error(f"Failed to connect to {self.uri}") return False
def process_item_pages(responses): """ Extracts barcodes and image urls from pages :param responses: List of (item, response) 2-tuples :return: List of dict item objects """ items = [] for item, res in responses: try: new_item = process_item_page(item, res) items.append(new_item) except Exception as err: tools.log_error(err) return items
def _create_price_list(stores, cd_items_dict, barcodes): """ Iterate through countdown items and create price data :param stores: List of countdown stores from pisspricer api :param cd_items_dict: Dictionary of countdown stores and items :param barcodes: Dictionary of barcodes from pisspricer api :return: List of (sku, store_id, payload) tuples """ # Create a dict of {"cd_id": "store_id"} store_dict = {} for store in stores: cd_id = store["internalId"] store_dict[cd_id] = store["storeId"] # Create a list of price data tuples price_data = [] for cd_id, cats in cd_items_dict.items(): for cat_obj in cats: items = cat_obj["items"] for item in items: try: # Create payload price_item = { "price": item["price"]["originalPrice"], "internalSku": item["sku"], } if item["price"]["isSpecial"]: price_item["salePrice"] = item["price"]["salePrice"] # Get storeId and sku store_id = store_dict[cd_id] sku = barcodes[item["barcode"]][0] # Add data to list price_data.append((sku, store_id, price_item,)) except Exception as err: tools.log_error(err) return price_data
def handle_command(self, data: str): message = json.loads(data) try: self.visit(message) except Exception: log_error(traceback.format_exc())
def _get_cd_items(self, stores): """ Gets all items from Countdown API :param stores: List of Countdown stores from pisspricer api :return: Dictionary of items {cd_id: [{'cat': str, 'subcat': str, 'items': []}] } """ task = "_get_cd_items" item_url = self.cd_base_url + self.cd_items + self.page_lim_str items_dict = dict() count = 0 self.print_progress(0, len(stores), task) for store in stores: try: # Set store and get first page items self._set_store(store["internalId"]) items_res = requests.get(item_url, headers=self.cd_headers, cookies=self.cookies) if not items_res.ok: raise CountdownApiException(items_res, task) items_json = items_res.json() # Generate url's cats = items_json["dasFacets"] urls = [] for cat in cats: cat_name = cat["name"] cat_count = cat["productCount"] url_end = f"&dasFilter=Aisle;;{cat_name.replace(' ', '-').replace('&', '')};false" if "wine" in cat_name: cat_info = { "cat": "Wine", "subcat": cat_name } else: cat_info = { "cat": cat_name, "subcat": None } urls += tools.generate_url_pages(item_url + "&page=", cat_count, self.page_lim, url_end=url_end, carry=cat_info) responses = tools.async_get_list(urls, headers=self.cd_headers, cookies=self.cookies) # Iterate through responses and make a list of data items = [] for res in responses: # Check if cat is already in list res_cat = res["carry"] cat_in_list = False index = -1 for i, cat in enumerate(items): if cat["cat"] == res_cat["cat"] and cat["subcat"] == res_cat["subcat"]: cat_in_list = True index = i break if cat_in_list: items[i]["items"] += res['products']['items'] else: items.append({"cat": res_cat["cat"], "subcat": res_cat["subcat"], "items": res['products']['items']}) # Assign items to dict items_dict[store["internalId"]] = items except Exception as err: tools.log_error(err) finally: count += 1 self.print_progress(count, len(stores), task) return items_dict
def _get_new_items(cd_items, barcodes, cur_cats): """ Take a dictionary of countdown items and return a list of items that aren't in the database. Categorys that aren't in database get added as it goes :param cd_items: Dictionary of countdown items :param barcodes: Dictionary of barcodes that are currently in the database :return: List of json items that aren't in the database """ new_items = [] new_barcodes = set() for store_id, cats in cd_items.items(): for cat_obj in cats: try: # Get cats cat = cat_obj["cat"].lower() subcat = None if cat_obj["subcat"] is None else cat_obj["subcat"].lower() # Create cat and subcat if they dont exist if cat not in cur_cats: # create cat cat_id = tools.post_category(cat) # Add to dict cur_cats[cat] = { "category": cat, "categoryId": cat_id, "subcategories": [] } else: cat_id = cur_cats[cat]["categoryId"] subcat_id = None if subcat is not None: # Subcat specified subcats = cur_cats[cat]["subcategories"] for sub in subcats: if sub["subcategory"] == subcat: subcat_id = sub["subcategoryId"] break if subcat_id is None: # Subcat needs to be created subcat_id = tools.post_subcategory(cat_id, subcat) cur_cats[cat]["subcategories"].append({ "subcategory": subcat, "subcategoryId": subcat_id }) items = cat_obj["items"] for item in items: if item["barcode"] not in barcodes and item["barcode"] not in new_barcodes: volume = item["size"]["volumeSize"] new_item = { "name": item["name"] + (" " + volume if volume is not None else ""), "brand": item["brand"], "barcode": item["barcode"], "categoryId": cat_id, "image_url": item["images"]["big"], } if subcat_id is not None: new_item["subcategoryId"] = subcat_id new_items.append(new_item) new_barcodes.add(item["barcode"]) except Exception as err: tools.log_error(err) # raise err return new_items
def update_all_items(self, debug=False): """ Updates all items for all known Countdown store """ task = "update_all_items" # Get all current countdown stores stores_res = requests.get(api.url + "/stores", headers=api.headers, params={"brandId": Countdown.cd_brand_id}) if not stores_res.ok: raise PisspricerApiException(stores_res, task) stores = stores_res.json() # Get a set of barcodes from pisspricer barcodes_res = requests.get(api.url + "/barcodes", headers=api.headers) tools.check_pisspricer_res(barcodes_res, task) barcodes = barcodes_res.json() # Iterate through stores and get items from countdown api cd_items_dict = self._get_cd_items(stores) # Get categories categories_res = requests.get(api.url + "/categories", headers=api.headers) tools.check_pisspricer_res(categories_res, task) categories = categories_res.json() # Get a list of new items new_items = self._get_new_items(cd_items_dict, barcodes, categories) # Async post all new items if len(new_items) != 0: self.print_progress(0, len(new_items), "upload new items") new_items_skus = tools.async_post_items(new_items, api.url + "/items", headers=api.headers, printer=(self.print_progress, len(new_items), "upload new items")) # Get a set of barcodes from pisspricer barcodes_res = requests.get(api.url + "/barcodes", headers=api.headers) tools.check_pisspricer_res(barcodes_res, task) barcodes = barcodes_res.json() # Upload images for new items pisspricer = Pisspricer(api) new_images_url = [] # Iterate through items and assign image dicts for cat_list in cd_items_dict.values(): for cat_dict in cat_list: items = cat_dict['items'] for item in items: try: image_url = item["images"]["big"] barcode = item["barcode"] sku = barcodes[barcode][0] new_images_url.append({"sku": sku, "image_url": image_url}) except Exception as err: tools.log_error(err) pisspricer.upload_new_images(new_images_url, self.print_progress) new_images = self._get_new_images(new_items, barcodes) # if len(new_images) != 0: # self.print_progress(0, len(new_images), "upload item images") # responses = custom_reqs.post_images(new_images, # f"{api.url}/items", # headers=api.headers, # printer=(self.print_progress, len(new_images), "upload item images")) # Put price data into pisspricer api prices_list = self._create_price_list(stores, cd_items_dict, barcodes) price_data_res = custom_reqs.put_prices(prices_list, api.url, headers=api.headers)
try: print '--------------------------------------------------------------------------------' error_log = "%s,%s,%s" % (row['id'],row['author'],row['href']) # print "error_log:::",error_log '''=====================代理模式=========================== ''' proxy = proxylist[random.randint(0,len(proxylist)-1)] print "***********************************************************" print proxy print "***********************************************************" proxies = {'': proxy} opener = urllib.FancyURLopener(proxies) f = opener.open(row['href']) soup= BeautifulSoup(f.read(), "lxml") '''=====================单机模式=========================== page = urllib.urlopen(row['href']) soup = BeautifulSoup( page.read(), "lxml") ''' main = soup.find(id="main") name_str=main.find(id="headline").find("span",class_="name primary").get_text() result = db.update_author(row['id'],name_str) #db.close() print '[',row['id'],']',result,"@@@",row['author'],"@@@",name_str,'@@@',row['href'] print '--------------------------------------------------------------------------------' except: mytools.log_error("log_error_update_author.log",['',error_log]) i=i+step print "Start : %s" % time.ctime() time.sleep(4) print "End : %s" % time.ctime() print "finish!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
def upload_new_stores(self, locations, brand_id, printer=None): """ Posts stores to pisspricer api that are new (based on internal id). Gets data from google maps api if location data is incomplete. Uploads region if it doesn't already exist. :param printer: (print_function, total, title) for printing :param brand_id: Brand id of store locations :param locations: List of dict objects { name: "required|string", url: "required|string", region: "required|string", region_lat: "numeric", region_lng: "numeric", address: "required|string", postcode: "numeric", lattitude: "numeric", longitude: "numeric" internalId: "string" } :return: None """ # Get current locations cur_locations_res = req.get(self.api.url + "/stores", headers=self.api.headers, params={"brandId": brand_id}) if cur_locations_res.status != 200: raise custom_exceptions.AiohttpException(cur_locations_res, "get stores", "pisspricer") cur_locations = cur_locations_res.json() # Create a set of internal ids cur_locs_set = set() for loc in cur_locations: cur_locs_set.add(loc["internalId"]) # Get a list of regions regions = self.get_regions() # Print first iteration if printer is not None: print_func, total, task = printer print_func(0, total, task) # Create a list of new stores new_locations = [] for i, loc in enumerate(locations): try: # Check if the store is new if loc["internalId"] not in cur_locs_set: # Get location if data not supplied region = loc["region"] lat = loc["lattitude"] lng = loc["longitude"] postcode = loc["postcode"] address = loc["address"] location_list = [region, lat, lng, postcode, address] if any(item is None for item in location_list): lat, lng, address, postcode, region = tools.geocode_address(f"{loc['name']}, {address}") # Create new location dict new_loc = { "name": loc["name"], "url": loc["url"], "brandId": brand_id, "regionId": self._get_region_id(regions, region, lat=loc["region_lat"], lng=loc["region_lng"]), "lattitude": lat, "longitude": lng, "postcode": postcode, "address": address, "internalId": loc["internalId"] } # Add new store to task list new_locations.append([self.api.url + "/stores", new_loc]) except custom_exceptions.GoogleApiException as err: tools.log_error(err) except custom_exceptions.AiohttpException as err: tools.log_error(err) finally: if printer is not None: print_func(i+1, total, task) # Post all stores # TODO Change post function iteration = [0] print_func, _, task = printer kwargs = {"headers": self.api.headers, "printer": (print_func, len(new_locations), task), "iteration": iteration} responses = asyncio.run(req.create_async_tasks(new_locations, kwargs, self._async_post_json)) for res in responses: if res.status != 201: tools.log_error(custom_exceptions.AiohttpException(res, "post stores", "pisspricer"))
error_log ="" try: print '--------------------------------------------------------------------------------' error_log = "%s,%s,%s" % (row['id'],row['author'],row['href']) # print "error_log:::",error_log '''=====================代理模式=========================== ''' proxy = proxylist[random.randint(0,len(proxylist)-1)] print "***********************************************************" print proxy print "***********************************************************" proxies = {'': proxy} opener = urllib.FancyURLopener(proxies) f = opener.open(row['href']) soup= BeautifulSoup(f.read(), "lxml") '''=====================单机模式=========================== page = urllib.urlopen(row['href']) soup = BeautifulSoup( page.read(), "lxml") ''' main = soup.find(id="main") name_str=main.find(id="headline").find("span",class_="name primary").get_text() result = db.update_author(row['id'],name_str) #db.close() print '[',row['id'],']',result,"@@@",row['author'],"@@@",name_str,'@@@',row['href'] print '--------------------------------------------------------------------------------' except: mytools.log_error("log_error_update_author.log",['',error_log]) i=i+step print "Start : %s" % time.ctime() time.sleep(4) print "End : %s" % time.ctime() print "finish!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# create the single options object, load options and send a log message t.load_options(Options_File) t.log_info('options loaded from user folder ({})'.format(User_Folder)) # between versions the format of the options might change, then it's better to overwrite old options if t.options[c.O_OPTIONS_VERSION] < c.Options_Version: t.log_warning('outdated version of options, have been overwritten') shutil.copyfile(c.Options_Default_File, Options_File) t.load_options(Options_File) # test for phonon availability if t.options[c.OM_PHONON_SUPPORTED]: try: from PySide.phonon import Phonon except ImportError: t.log_error('Phonon backend not available, no sound.') t.options[c.OM_PHONON_SUPPORTED] = False # special case of some desktop environments under Linux where full screen mode does not work well if t.options[c.OG_FULLSCREEN_SUPPORTED]: desktop_session = os.environ.get("DESKTOP_SESSION") if desktop_session and (desktop_session.startswith('ubuntu') or 'xfce' in desktop_session or desktop_session.startswith('xubuntu') or 'gnome' in desktop_session): t.options[c.OG_FULLSCREEN_SUPPORTED] = False t.log_warning( 'Desktop environment {} has problems with full screen mode. Will turn if off.'.format(desktop_session)) if not t.options[c.OG_FULLSCREEN_SUPPORTED]: t.options[c.OG_MW_FULLSCREEN] = False # now we can safely assume that the environment is good to us # and we simply start the client
def discard(self): log_error("Discarded {:02x}".format(self.buffer[0])) self.buffer = []
def upload_new_images(self, items, print_func): """ Uploads images for items that don't have one :param items: List of item dict objects. {"sku": int, "image_url": string} :return: None """ all_items = req.get(self.api.url + "/allitems", headers=self.api.headers).json() print('start') # Iterate through items and add to get list if there isn't an image req_list = [] checked_skus = set() for item in items: if item["sku"] not in checked_skus: has_image = True for cur_item in all_items: if cur_item["sku"] == item["sku"]: # Check if item doesn't have an image if cur_item["hasImage"] == 0: has_image = False break if not has_image: # Item doesn't have image req_list.append([ item["image_url"], item ]) # Add sku to set checked_skus.add(item["sku"]) # Get images iteration = [0] if len(req_list) > 0: print_func(0, len(req_list), "get images") responses = asyncio.run( req.create_async_tasks(req_list, {"headers": self.api.headers, "printer": (print_func, len(req_list), "get images"), "iteration": iteration}, self._async_get)) # Create put image request list image_list = [] print_func(0, len(responses), 'processing images') for i, (item, res) in enumerate(responses): try: content = res.read() image_bytes = images.process_response_content(content) image_list.append((item["sku"], image_bytes)) except Exception as err: tools.log_error(err) finally: print_func(i + 1, len(responses)) if len(image_list) > 0: print_func(0, len(image_list), "put images") responses = req.post_images(image_list, self.api.url + "/items", headers=self.api.headers, printer=(print_func, len(image_list), "put images"))