async def async_post(session,
url,
payload,
headers={},
cookies={},
printer=None,
iteration=None):
"""
Execute http post request
:param session: Session for requests
:param url: Url for http requests
:param payload: json payload for request
:param headers: HTTP Header for request
:param cookies: HTTP cookies for request
:param printer: (print_function, total, title) for printing
:param iteration: [int] list with int for counting print number
:return: None
"""
async with session.post(url,
headers=headers,
cookies=cookies,
json=payload) as response:
try:
if printer is not None:
print_function, total, task = printer
iteration[0] += 1
print_function(iteration[0], total, task)
if response.status != 201:
raise Exception("Error while posting item: " + str(response))
resp = await response.json()
return resp
except Exception as err:
tools.log_error(err)
raise err
def process_henry_items(items):
"""
Processes a list of items from henrys api, converts into pisspricer relevant info
:param items: List of items from henrys api
:return: List of items for pisspricer api
[
{
"name": "string",
"brand": "string",
"categoryId": 0,
"subcategoryId": 0,
"stdDrinks": 0,
"alcoholContent": 0,
"volumeEach": 0,
"barcode": "string",
"salePrice": 0,
"price": 0,
"stock": "string",
"internalSku": "string",
"url": "string"
}
]
"""
new_items = []
# Get most item details
for henry_item in items:
try:
new_item = process_henry_item(henry_item)
new_items.append(new_item)
except Exception as err:
tools.log_error(err)
print(items[0])
return new_items
def process_stores_page(page_html):
"""
Processes henrys stores page into list of dictionary store items
:param page_html: String of page html
:return: List of store dictionary items
{
name: "required|string",
url: "required|string",
region: "required|string",
region_lat: "numeric",
region_lng: "numeric",
address: "required|string",
postcode: "numeric",
lattitude: "numeric",
longitude: "numeric"
internalId: "string"
}
"""
# Parse the html
soup = BeautifulSoup(page_html, features="html.parser")
# Get the store info json
store_json_list = json.loads(soup.find('store-locations')[':locations'])
stores = []
for store_json in store_json_list:
try:
store = process_store_json(store_json)
stores.append(store)
except Exception as err:
tools.log_error(err)
return stores
def handle_message(self, data: str):
message = json.loads(data)
try:
self.visit(message)
except NodeNotFoundException as e:
log_error(
f"Node not found: home ID = {e.home_id}, node ID = {e.node_id}"
)
def _post_stores(post_list, regions):
""" Iterate through store locations and post to api
@param post_list - List of stores from Countdown API to be posted to pisspricer api
@param regions - List of regions from pisspricer API
"""
Countdown.print_progress(0, len(post_list), title=Countdown.name + " Inserting Stores")
i = 0
for store_loc in post_list:
try:
# Get lat and lng
address_string = store_loc["name"] + ", " + store_loc["address"] + ", New Zealand"
lat, lng, address, postcode, region = tools.geocode_address(address_string)
# Set regionId
region_id = None
for reg in regions:
if reg["name"] == region:
region_id = reg["regionId"]
if region_id is None:
new_region_res = requests.post(api.url + "/regions",
headers=api.headers,
json={"name": region})
if not new_region_res.ok:
raise PisspricerApiException(new_region_res, f"posting to /regions {region}")
region_id = new_region_res.json()["regionId"]
# Set store values
store = {
"name": store_loc["name"],
"url": Countdown.store_url,
"brandId": Countdown.cd_brand_id,
"lattitude": lat,
"longitude": lng,
"regionId": region_id,
"address": address,
"postcode": postcode,
"internalId": str(store_loc["id"])
}
# Post new store
new_store_res = requests.post(api.url + '/stores',
headers=api.headers,
json=store)
if not new_store_res.ok:
raise PisspricerApiException(new_store_res, f"posting to /stores {store}")
except ApiException as err:
tools.log_error(err)
finally:
i += 1
Countdown.print_progress(i, len(post_list))
if i > 1000:
break
async def connect(self) -> bool:
try:
self.websocket = await websockets.connect(self.uri)
asyncio.create_task(self.start_polling())
log_info(f"Connected to {self.uri}")
return True
except ConnectionRefusedError:
log_error(f"Failed to connect to {self.uri}")
return False
def process_item_pages(responses):
"""
Extracts barcodes and image urls from pages
:param responses: List of (item, response) 2-tuples
:return: List of dict item objects
"""
items = []
for item, res in responses:
try:
new_item = process_item_page(item, res)
items.append(new_item)
except Exception as err:
tools.log_error(err)
return items
def _create_price_list(stores, cd_items_dict, barcodes):
"""
Iterate through countdown items and create price data
:param stores: List of countdown stores from pisspricer api
:param cd_items_dict: Dictionary of countdown stores and items
:param barcodes: Dictionary of barcodes from pisspricer api
:return: List of (sku, store_id, payload) tuples
"""
# Create a dict of {"cd_id": "store_id"}
store_dict = {}
for store in stores:
cd_id = store["internalId"]
store_dict[cd_id] = store["storeId"]
# Create a list of price data tuples
price_data = []
for cd_id, cats in cd_items_dict.items():
for cat_obj in cats:
items = cat_obj["items"]
for item in items:
try:
# Create payload
price_item = {
"price": item["price"]["originalPrice"],
"internalSku": item["sku"],
}
if item["price"]["isSpecial"]:
price_item["salePrice"] = item["price"]["salePrice"]
# Get storeId and sku
store_id = store_dict[cd_id]
sku = barcodes[item["barcode"]][0]
# Add data to list
price_data.append((sku, store_id, price_item,))
except Exception as err:
tools.log_error(err)
return price_data
def handle_command(self, data: str):
message = json.loads(data)
try:
self.visit(message)
except Exception:
log_error(traceback.format_exc())
def _get_cd_items(self, stores):
"""
Gets all items from Countdown API
:param stores: List of Countdown stores from pisspricer api
:return: Dictionary of items {cd_id: [{'cat': str, 'subcat': str, 'items': []}] }
"""
task = "_get_cd_items"
item_url = self.cd_base_url + self.cd_items + self.page_lim_str
items_dict = dict()
count = 0
self.print_progress(0, len(stores), task)
for store in stores:
try:
# Set store and get first page items
self._set_store(store["internalId"])
items_res = requests.get(item_url,
headers=self.cd_headers,
cookies=self.cookies)
if not items_res.ok:
raise CountdownApiException(items_res, task)
items_json = items_res.json()
# Generate url's
cats = items_json["dasFacets"]
urls = []
for cat in cats:
cat_name = cat["name"]
cat_count = cat["productCount"]
url_end = f"&dasFilter=Aisle;;{cat_name.replace(' ', '-').replace('&', '')};false"
if "wine" in cat_name:
cat_info = {
"cat": "Wine",
"subcat": cat_name
}
else:
cat_info = {
"cat": cat_name,
"subcat": None
}
urls += tools.generate_url_pages(item_url + "&page=", cat_count, self.page_lim,
url_end=url_end, carry=cat_info)
responses = tools.async_get_list(urls, headers=self.cd_headers, cookies=self.cookies)
# Iterate through responses and make a list of data
items = []
for res in responses:
# Check if cat is already in list
res_cat = res["carry"]
cat_in_list = False
index = -1
for i, cat in enumerate(items):
if cat["cat"] == res_cat["cat"] and cat["subcat"] == res_cat["subcat"]:
cat_in_list = True
index = i
break
if cat_in_list:
items[i]["items"] += res['products']['items']
else:
items.append({"cat": res_cat["cat"],
"subcat": res_cat["subcat"],
"items": res['products']['items']})
# Assign items to dict
items_dict[store["internalId"]] = items
except Exception as err:
tools.log_error(err)
finally:
count += 1
self.print_progress(count, len(stores), task)
return items_dict
def _get_new_items(cd_items, barcodes, cur_cats):
"""
Take a dictionary of countdown items and return a list of items that aren't in the database.
Categorys that aren't in database get added as it goes
:param cd_items: Dictionary of countdown items
:param barcodes: Dictionary of barcodes that are currently in the database
:return: List of json items that aren't in the database
"""
new_items = []
new_barcodes = set()
for store_id, cats in cd_items.items():
for cat_obj in cats:
try:
# Get cats
cat = cat_obj["cat"].lower()
subcat = None if cat_obj["subcat"] is None else cat_obj["subcat"].lower()
# Create cat and subcat if they dont exist
if cat not in cur_cats:
# create cat
cat_id = tools.post_category(cat)
# Add to dict
cur_cats[cat] = {
"category": cat,
"categoryId": cat_id,
"subcategories": []
}
else:
cat_id = cur_cats[cat]["categoryId"]
subcat_id = None
if subcat is not None:
# Subcat specified
subcats = cur_cats[cat]["subcategories"]
for sub in subcats:
if sub["subcategory"] == subcat:
subcat_id = sub["subcategoryId"]
break
if subcat_id is None:
# Subcat needs to be created
subcat_id = tools.post_subcategory(cat_id, subcat)
cur_cats[cat]["subcategories"].append({
"subcategory": subcat,
"subcategoryId": subcat_id
})
items = cat_obj["items"]
for item in items:
if item["barcode"] not in barcodes and item["barcode"] not in new_barcodes:
volume = item["size"]["volumeSize"]
new_item = {
"name": item["name"] + (" " + volume if volume is not None else ""),
"brand": item["brand"],
"barcode": item["barcode"],
"categoryId": cat_id,
"image_url": item["images"]["big"],
}
if subcat_id is not None:
new_item["subcategoryId"] = subcat_id
new_items.append(new_item)
new_barcodes.add(item["barcode"])
except Exception as err:
tools.log_error(err)
# raise err
return new_items
def update_all_items(self, debug=False):
""" Updates all items for all known Countdown store """
task = "update_all_items"
# Get all current countdown stores
stores_res = requests.get(api.url + "/stores",
headers=api.headers,
params={"brandId": Countdown.cd_brand_id})
if not stores_res.ok:
raise PisspricerApiException(stores_res, task)
stores = stores_res.json()
# Get a set of barcodes from pisspricer
barcodes_res = requests.get(api.url + "/barcodes",
headers=api.headers)
tools.check_pisspricer_res(barcodes_res, task)
barcodes = barcodes_res.json()
# Iterate through stores and get items from countdown api
cd_items_dict = self._get_cd_items(stores)
# Get categories
categories_res = requests.get(api.url + "/categories", headers=api.headers)
tools.check_pisspricer_res(categories_res, task)
categories = categories_res.json()
# Get a list of new items
new_items = self._get_new_items(cd_items_dict, barcodes, categories)
# Async post all new items
if len(new_items) != 0:
self.print_progress(0, len(new_items), "upload new items")
new_items_skus = tools.async_post_items(new_items,
api.url + "/items",
headers=api.headers,
printer=(self.print_progress, len(new_items), "upload new items"))
# Get a set of barcodes from pisspricer
barcodes_res = requests.get(api.url + "/barcodes",
headers=api.headers)
tools.check_pisspricer_res(barcodes_res, task)
barcodes = barcodes_res.json()
# Upload images for new items
pisspricer = Pisspricer(api)
new_images_url = []
# Iterate through items and assign image dicts
for cat_list in cd_items_dict.values():
for cat_dict in cat_list:
items = cat_dict['items']
for item in items:
try:
image_url = item["images"]["big"]
barcode = item["barcode"]
sku = barcodes[barcode][0]
new_images_url.append({"sku": sku, "image_url": image_url})
except Exception as err:
tools.log_error(err)
pisspricer.upload_new_images(new_images_url, self.print_progress)
new_images = self._get_new_images(new_items, barcodes)
# if len(new_images) != 0:
# self.print_progress(0, len(new_images), "upload item images")
# responses = custom_reqs.post_images(new_images,
# f"{api.url}/items",
# headers=api.headers,
# printer=(self.print_progress, len(new_images), "upload item images"))
# Put price data into pisspricer api
prices_list = self._create_price_list(stores, cd_items_dict, barcodes)
price_data_res = custom_reqs.put_prices(prices_list,
api.url,
headers=api.headers)
try:
print '--------------------------------------------------------------------------------'
error_log = "%s,%s,%s" % (row['id'],row['author'],row['href'])
# print "error_log:::",error_log
'''=====================代理模式=========================== '''
proxy = proxylist[random.randint(0,len(proxylist)-1)]
print "***********************************************************"
print proxy
print "***********************************************************"
proxies = {'': proxy}
opener = urllib.FancyURLopener(proxies)
f = opener.open(row['href'])
soup= BeautifulSoup(f.read(), "lxml")
'''=====================单机模式===========================
page = urllib.urlopen(row['href'])
soup = BeautifulSoup( page.read(), "lxml") '''
main = soup.find(id="main")
name_str=main.find(id="headline").find("span",class_="name primary").get_text()
result = db.update_author(row['id'],name_str)
#db.close()
print '[',row['id'],']',result,"@@@",row['author'],"@@@",name_str,'@@@',row['href']
print '--------------------------------------------------------------------------------'
except:
mytools.log_error("log_error_update_author.log",['',error_log])
i=i+step
print "Start : %s" % time.ctime()
time.sleep(4)
print "End : %s" % time.ctime()
print "finish!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
def upload_new_stores(self, locations, brand_id, printer=None):
"""
Posts stores to pisspricer api that are new (based on internal id).
Gets data from google maps api if location data is incomplete.
Uploads region if it doesn't already exist.
:param printer: (print_function, total, title) for printing
:param brand_id: Brand id of store locations
:param locations: List of dict objects
{
name: "required|string",
url: "required|string",
region: "required|string",
region_lat: "numeric",
region_lng: "numeric",
address: "required|string",
postcode: "numeric",
lattitude: "numeric",
longitude: "numeric"
internalId: "string"
}
:return: None
"""
# Get current locations
cur_locations_res = req.get(self.api.url + "/stores",
headers=self.api.headers,
params={"brandId": brand_id})
if cur_locations_res.status != 200:
raise custom_exceptions.AiohttpException(cur_locations_res, "get stores", "pisspricer")
cur_locations = cur_locations_res.json()
# Create a set of internal ids
cur_locs_set = set()
for loc in cur_locations:
cur_locs_set.add(loc["internalId"])
# Get a list of regions
regions = self.get_regions()
# Print first iteration
if printer is not None:
print_func, total, task = printer
print_func(0, total, task)
# Create a list of new stores
new_locations = []
for i, loc in enumerate(locations):
try:
# Check if the store is new
if loc["internalId"] not in cur_locs_set:
# Get location if data not supplied
region = loc["region"]
lat = loc["lattitude"]
lng = loc["longitude"]
postcode = loc["postcode"]
address = loc["address"]
location_list = [region, lat, lng, postcode, address]
if any(item is None for item in location_list):
lat, lng, address, postcode, region = tools.geocode_address(f"{loc['name']}, {address}")
# Create new location dict
new_loc = {
"name": loc["name"],
"url": loc["url"],
"brandId": brand_id,
"regionId": self._get_region_id(regions, region, lat=loc["region_lat"], lng=loc["region_lng"]),
"lattitude": lat,
"longitude": lng,
"postcode": postcode,
"address": address,
"internalId": loc["internalId"]
}
# Add new store to task list
new_locations.append([self.api.url + "/stores", new_loc])
except custom_exceptions.GoogleApiException as err:
tools.log_error(err)
except custom_exceptions.AiohttpException as err:
tools.log_error(err)
finally:
if printer is not None:
print_func(i+1, total, task)
# Post all stores
# TODO Change post function
iteration = [0]
print_func, _, task = printer
kwargs = {"headers": self.api.headers,
"printer": (print_func, len(new_locations), task),
"iteration": iteration}
responses = asyncio.run(req.create_async_tasks(new_locations, kwargs, self._async_post_json))
for res in responses:
if res.status != 201:
tools.log_error(custom_exceptions.AiohttpException(res, "post stores", "pisspricer"))
error_log =""
try:
print '--------------------------------------------------------------------------------'
error_log = "%s,%s,%s" % (row['id'],row['author'],row['href'])
# print "error_log:::",error_log
'''=====================代理模式=========================== '''
proxy = proxylist[random.randint(0,len(proxylist)-1)]
print "***********************************************************"
print proxy
print "***********************************************************"
proxies = {'': proxy}
opener = urllib.FancyURLopener(proxies)
f = opener.open(row['href'])
soup= BeautifulSoup(f.read(), "lxml")
'''=====================单机模式===========================
page = urllib.urlopen(row['href'])
soup = BeautifulSoup( page.read(), "lxml") '''
main = soup.find(id="main")
name_str=main.find(id="headline").find("span",class_="name primary").get_text()
result = db.update_author(row['id'],name_str)
#db.close()
print '[',row['id'],']',result,"@@@",row['author'],"@@@",name_str,'@@@',row['href']
print '--------------------------------------------------------------------------------'
except:
mytools.log_error("log_error_update_author.log",['',error_log])
i=i+step
print "Start : %s" % time.ctime()
time.sleep(4)
print "End : %s" % time.ctime()
print "finish!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# create the single options object, load options and send a log message
t.load_options(Options_File)
t.log_info('options loaded from user folder ({})'.format(User_Folder))
# between versions the format of the options might change, then it's better to overwrite old options
if t.options[c.O_OPTIONS_VERSION] < c.Options_Version:
t.log_warning('outdated version of options, have been overwritten')
shutil.copyfile(c.Options_Default_File, Options_File)
t.load_options(Options_File)
# test for phonon availability
if t.options[c.OM_PHONON_SUPPORTED]:
try:
from PySide.phonon import Phonon
except ImportError:
t.log_error('Phonon backend not available, no sound.')
t.options[c.OM_PHONON_SUPPORTED] = False
# special case of some desktop environments under Linux where full screen mode does not work well
if t.options[c.OG_FULLSCREEN_SUPPORTED]:
desktop_session = os.environ.get("DESKTOP_SESSION")
if desktop_session and (desktop_session.startswith('ubuntu') or 'xfce' in desktop_session
or desktop_session.startswith('xubuntu') or 'gnome' in desktop_session):
t.options[c.OG_FULLSCREEN_SUPPORTED] = False
t.log_warning(
'Desktop environment {} has problems with full screen mode. Will turn if off.'.format(desktop_session))
if not t.options[c.OG_FULLSCREEN_SUPPORTED]:
t.options[c.OG_MW_FULLSCREEN] = False
# now we can safely assume that the environment is good to us
# and we simply start the client
def discard(self):
log_error("Discarded {:02x}".format(self.buffer[0]))
self.buffer = []
def upload_new_images(self, items, print_func):
"""
Uploads images for items that don't have one
:param items: List of item dict objects. {"sku": int, "image_url": string}
:return: None
"""
all_items = req.get(self.api.url + "/allitems", headers=self.api.headers).json()
print('start')
# Iterate through items and add to get list if there isn't an image
req_list = []
checked_skus = set()
for item in items:
if item["sku"] not in checked_skus:
has_image = True
for cur_item in all_items:
if cur_item["sku"] == item["sku"]:
# Check if item doesn't have an image
if cur_item["hasImage"] == 0:
has_image = False
break
if not has_image:
# Item doesn't have image
req_list.append([
item["image_url"],
item
])
# Add sku to set
checked_skus.add(item["sku"])
# Get images
iteration = [0]
if len(req_list) > 0:
print_func(0, len(req_list), "get images")
responses = asyncio.run(
req.create_async_tasks(req_list,
{"headers": self.api.headers,
"printer": (print_func, len(req_list), "get images"),
"iteration": iteration},
self._async_get))
# Create put image request list
image_list = []
print_func(0, len(responses), 'processing images')
for i, (item, res) in enumerate(responses):
try:
content = res.read()
image_bytes = images.process_response_content(content)
image_list.append((item["sku"], image_bytes))
except Exception as err:
tools.log_error(err)
finally:
print_func(i + 1, len(responses))
if len(image_list) > 0:
print_func(0, len(image_list), "put images")
responses = req.post_images(image_list,
self.api.url + "/items",
headers=self.api.headers,
printer=(print_func, len(image_list), "put images"))
