def get_genre_refs(*genres):
genre_objs=[]
for agenre_obj in genres:
#find all matching genre
if isinstance(agenre_obj,str):
genre_models=Genre.objects(genre=agenre_obj)
else:
genre_models=Genre.objects(genre=agenre_obj['genre'])
if(len(genre_models)==0):
if not isinstance(agenre_obj,str):
genre_model=Genre()
for (k,v) in agenre_obj.items():
genre_model[k]=v
else:
genre_model=Genre(genre=agenre_obj)
try:
genre_model.save()
except:
Logger.error("Error saving: "+str(agenre_obj))
genre_objs.append(genre_model)
else:
genre_objs.extend(genre_models.all())
return genre_objs
def scrape(self,url,parent):
Logger.debug('Starting url scrap for {}'.format(url))
config.last_url_and_parent=url+', {}'.format('' if parent==None else parent)
new_url=base_util.unreplace_dot_url(url)
response=self.http.get(new_url)
Logger.debug('Got URL')
if not hasattr(response,'data') and new_url.startswith('www.'):
new_url=new_url.replace('www.','http://')
response=self.http.get(new_url)
if not hasattr(response,'data'):
new_url=new_url.replace('http://','http://www.')
response=self.http.get(new_url)
if hasattr(response,'data'):
body=base_util.utf_8_safe_decode(response.data)
else:
Logger.error('No data associated with '+new_url)
raise AttributeError(new_url+':::No data')
return body,new_url
class GetAuth:
def __init__(self, env='QA'):
self.log = Logger("debug")
self.opera = OperationIni(fileName='config.ini', pathName='config')
path = '/website/saas/account/api2/user/login'
self.key = env.lower() + '_token'
d = get_env_authorization(env=env)
self.url = d[0] + path
self.cookie = d[1]
self.userName = d[2]
self.passWord = d[3]
# if env == 'QA':
# self.url = self.opera.read_ini(section='Authorization', key='qa_url') + path
# self.cookie = self.opera.read_ini(section='Authorization', key='qa_cookie')
# self.userName = self.opera.read_ini(section='Authorization', key='qa_username')
# self.passWord = self.opera.read_ini(section='Authorization', key='qa_password')
# if env == 'DEV':
# self.url = self.opera.read_ini(section='Authorization', key='dev_url') + path
# self.cookie = self.opera.read_ini(section='Authorization', key='dev_cookie')
# self.userName = self.opera.read_ini(section='Authorization', key='dev_username')
# self.passWord = self.opera.read_ini(section='Authorization', key='dev_password')
self.headers = {
'Cookie': self.cookie,
'Content-Type': 'application/x-www-form-urlencoded'
}
def get_auth(self):
'''
获取token
:return: rsp, 登录后的token
'''
data = {
'zone': '0086',
'phone': self.userName,
'password': self.passWord,
'remember': False,
'passwordType': 'new'
}
self.log.info('开始：调用获取B端后台token接口，请求地址为：{0}，入参为：{1}，请求头为：{2}'.format(
self.url, data, self.headers))
r = requests.post(url=self.url, data=data, headers=self.headers)
try:
token = r.json()['data']['token']
self.log.info('结束：调用获取B端后台token接口，获取到token为：{0}'.format(token))
return token, r.json()
except Exception as f:
self.log.error('获取B端后台token失败，错误日志为：{0}'.format(f))
print(f)
def set_auth(self):
'''
存储最新的后端Authorization
:return:
'''
token = self.get_auth()[0]
self.opera.write_ini(section='Authorization', data=token, key=self.key)
def save_url(**kwargs):
kwargs['url']=replace_dot_url(kwargs['url'])
url_model=URLToGenre(**kwargs)
try:
save_obj=url_model.save()
except:
Logger.error("Error saving: "+str(kwargs['url']))
return save_obj
def push_to_queue(number,url_doc):
try:
if URLQueue.objects(number=number):
return None
URLQueue(number=number,document=url_doc).save()
except:
try:
Logger.error('Failed to save with url: {}'.format(url_doc['url']))
except:
Logger.error('Complete error to save number: {}'+number)
def __onConsume(self):
try:
Logger.info("Initializing MQTT Response ...")
self.__client = mqtt.Client()
self.__client.on_connect = self.__onConnect
self.__client.on_message = self.__onMessage
broker = self.__properties.get('address.broker')
port = int(self.__properties.get('port.broker'))
keepAliveBroker = int(self.__properties.get('keep.alive.broker'))
subscribe = self.__properties.get('topic.subscribe.broker')
self.__client.connect(broker, port, keepAliveBroker)
self.__client.subscribe(subscribe)
self.__client.loop_forever()
except Exception as exception:
Logger.error("MQTT Response failed. Cause: " + str(exception))
def scrape_urls_multiproc(cls):
#current position
pos=MongoDB.get(MetaData,'position',type='queue')
#current cap
cap=pos
process_queue=queue.Queue(maxsize=settings.NUM_PROCESSES)
#creates all the necessary processes
for p_num in range(0,settings.NUM_PROCESSES):
p=mp.Process(target=WebScraper().scrape_links_from_position,args=[cap])
#get curresponding objects
process_queue.put(p)
cap+=settings.NUM_URLS_PER_PROCESS
#now start
p.start()
head=process_queue.get()
#wait and create new processes as needed
while(pos<MongoDB.count(URLQueue)):
head.join()
if not head.exitcode ==0:
Logger.error('Error with Process, terminating')
return
#update counter
MongoDB.increment_url_counter(settings.NUM_URLS_PER_PROCESS)
p=mp.Process(target=WebScraper().scrape_links_from_position,args=[cap])
process_queue.put(p)
p.start()
#increase both cap and current position
cap+=settings.NUM_URLS_PER_PROCESS
pos+=settings.NUM_URLS_PER_PROCESS
head=process_queue.get()
print(p.exitcode)
return cls
def scrape(self):
home=self.http.get(dmoz_home)
home_page_links=self._scrapeHomeAndGetLinks(home.data)
#visit each link in homepage and dig down
#for url in home_page_links:
i=0
while i<settings.NUM_RANDOM_WEBPAGE:
result=self._scrapPage(home_page_links[random.randint(0,len(home_page_links)-1)])
if result is not None and MongoDB.get_url_object(result['url']) is None:
i+=1
try:
page=utf_8_safe_decode(self.http.get(result['url']).data)
MongoDB.save_modify_url(page=page,**result)
Logger.info("Completed: "+result['url'])
except Exception as ex:
Logger.error(ex)
class updateGoodsPrice:
def __init__(self, env='QA'):
self.log = Logger("debug")
self.opera = OperationIni(fileName='config.ini', pathName='config')
self.get_skuId = GetGoodsDetail(env=env)
self.get_access_token = GetAccessToken(env=env)
# env字符串转小写
env = env.lower()
key = env + '_url'
self.base_url = self.opera.read_ini(section='goods', key=key)
self.path = self.opera.read_ini(section='goods', key='updatePrice')
self.access_token = self.get_access_token.get_ini_access_token()
def update_goods_price(self, storeId, goodsId, originalPrice, salePrice):
'''
修改商品价格
:param storeId: 门店id
:param goodsId: 商品id
:param originalPrice: 市场价
:param salePrice: 商家统一价
:return: rsq
'''
url = self.base_url.format(self.path, self.access_token)
# 获取skuId
try:
self.skuId = self.get_skuId.get_goods_detail(goodsId, storeId)[1]
json_data = {
'goodsId':
goodsId,
'skuList': [{
'skuId': self.skuId,
'originalPrice': originalPrice,
'salePrice': salePrice,
}],
'storeId':
storeId
}
self.log.info('开始：调用update_goods_price方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
r = requests.post(url=url, json=json_data)
# 如果access_token无效
if r.json()['data'] == 'invalid accesstoken':
self.log.warning('提示：ini文件中的accesstoken失效，开始获取新的accesstoken')
# 获取最新的token并存入ini文件
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
url = self.base_url.format(self.path, new_access_token)
self.log.warning(
'开始：调用update_goods_price方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
res = requests.post(url=url, json=json_data)
self.log.warning('结束：调用update_goods_price方法，返回数据为:{0}'.format(
res.json()))
return res.json()
else:
self.log.info('结束：调用update_goods_price方法，返回数据为:{0}'.format(
r.json()))
return r.json()
except Exception as f:
self.log.error('修改商品失败，错误日志为：{0}'.format(f))
return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
class MockOrderThrow:
def __init__(self, pid, env):
self.log = Logger("debug")
self.pid = pid
self.env = env
self.opera = OperationIni()
self.get_order_detail = GetOrderDetail(pid=pid, env=env)
def mock_order_throw(self, orderNo):
url = self.opera.read_ini(self.env, key='mock_order_throw_ip')
mock_url = 'http://' + url + ':8080/service'
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
order_detail = self.get_order_detail.get_order_item_id_skuNum(
orderNo=orderNo)
pickingPackageList = order_detail[0]
storeId = order_detail[1]
wid = order_detail[2]
# 组装paramterInput参数
paramterInput = [{
"markNo": "111",
"orderNo": orderNo,
"pickingPackageList": pickingPackageList,
"pid": self.pid,
"storeId": storeId,
"wid": wid
}]
# 组装参数
data = {
'serviceName': 'orderCenterUpdateExportService',
'methodName': 'pickingAndDelivery',
'paramterInput': '{0}'.format(paramterInput)
}
# 连接VPN
start_vpn()
code = None
try:
self.log.info("开始:调用订单抛出服务接口，请求地址为：{0}，入参为：{1}，请求头为：{2}".format(
mock_url, data, headers))
r = requests.post(url=mock_url,
data=data,
headers=headers,
timeout=3)
code = r.status_code
result = r.json()
print('我要看:{0}'.format(result))
self.log.info("结束:调用订单抛出服务接口，返回数据打印:{0}".format(result))
# 关闭VPN
stop_vpn()
return result
except Exception as f:
print(f)
status = False
# print(status)
if status == False or code != 200:
self.log.warning('IP已失效，重新获取IP')
url = GetSoaServerIp(
env=self.env,
serviceName='mock_order_throw_servicename').get_soa_url()
self.log.warning("获取的新IP为:{0}".format(url))
self.opera.write_ini(section=self.env,
data=url,
key='mock_order_throw_ip')
mock_url = 'http://' + url + ':8080/service'
self.log.warning("请求url为:{0}，请求data为:{1}，请求头为:{2}".format(
mock_url, data, headers))
try:
self.log.warning(
"开始:调用订单抛出服务接口，请求地址为：{0}，入参为：{1}，请求头为：{2}".format(
mock_url, data, headers))
r = requests.post(url=mock_url, data=data, headers=headers)
result = r.json()
self.log.warning("结束:调用订单抛出服务接口，返回数据打印:{0}".format(result))
# 关闭VPN
stop_vpn()
return result
except Exception as f:
msg = {'msg': '发生未知错误,请联系管理员,错误日志为:{0}'.format(f)}
self.log.error('发生未知错误,请联系管理员,错误日志为:{0}'.format(f))
# 关闭VPN
stop_vpn()
return msg
# g = MockOrderThrow(pid=1,env='QA')
# g.mock_order_throw(orderNo='10094010113')
class updateGoodsStock:
def __init__(self, env='QA'):
self.log = Logger("debug")
opera = OperationIni(fileName='config.ini', pathName='config')
self.get_skuId = GetGoodsDetail(env=env)
self.get_access_token = GetAccessToken(env=env)
# env字符串转小写
env = env.lower()
key = env + '_url'
self.base_url = opera.read_ini(section='goods', key=key)
self.path = opera.read_ini(section='goods', key='wholeUpdateStock')
self.access_token = self.get_access_token.get_ini_access_token()
# if env == 'QA':
# self.access_token = opera.read_ini(section='access_token', key='qa_access_token')
# if env == 'DEV':
# self.access_token = opera.read_ini(section='access_token', key='dev_access_token')
def update_goods_stock(self, goodsId, editStockNum, storeId=None):
'''
修改商品库存
:param goodsId: 商品id
:param editStockNum: 需要修改的库存
:param storeId: 门店id
:return: rsq
'''
url = self.base_url.format(self.path, self.access_token)
# 获取skuId
try:
self.skuId = self.get_skuId.get_goods_detail(goodsId, storeId)[1]
json_data = {
'goodsId': goodsId,
'storeId': storeId,
'skuList': [{
'skuId': self.skuId,
'editStockNum': editStockNum
}]
}
self.log.info('开始：调用update_goods_stock方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
r = requests.post(url=url, json=json_data)
# 如果access_token无效
if r.json()['data'] == 'invalid accesstoken':
# 获取最新的token并存入ini文件
self.log.warning('提示：ini文件中的accesstoken失效，开始获取新的accesstoken')
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
url = self.base_url.format(self.path, new_access_token)
self.log.warning(
'开始：调用update_goods_stock方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
res = requests.post(url=url, json=json_data)
self.log.warning('结束：调用update_goods_stock方法，返回数据为:{0}'.format(
res.json()))
return res.json()
else:
self.log.info('结束：调用update_goods_stock方法，返回数据为:{0}'.format(
r.json()))
return r.json()
except Exception as f:
self.log.error('调用获取商品详情接口失败，错误日志为：{0}'.format(f))
return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
def collect_bad_url():
"""
Make bows of websites in the bad url list
:return:
"""
queue=DBQueue_old("genre_bow")
#don't trust anything
summarizer=Summarizer()
bow=BagOfWords()
short_genre_to_genre=coll.ShortGenre()
url_to_bow=coll.URLBow()
start_pos=queue.get()
for c,line in enumerate(open("bad_url_summarize_bow.txt")):
if c<start_pos:
continue
url=line.split(" ")[1].split(":::")[0]
try:
print('New url {} num: {}'.format(url,c))
url_obj=coll.URLToGenre().select(url=url).find_one()
if not hasattr(url_obj,"original") or not url_obj["original"]:
print("Not original")
continue
#request page anyways, most of the bad pages are due to bad pagess
data=Request().get_data(base_util.unreplace_dot_url(base_util.unreplace_dot_url(url_obj["url"])))
if data is None:
raise Exception('url {} No has page'.format(url))
else:
if not hasattr(url_obj,"page") or len(data)>len(url_obj["page"]):
print("updating data")
data=base_util.utf_8_safe_decode(data)
if not hasattr(url_obj,"page"):
#save page if the new page is significantly bigger than the old one
url_obj.save(page=data)
else:
url_obj.update(page=data)
url_obj.reload()
if len(data) > len(url_obj.page):
raise Exception("Inconsistency b/w data and page data")
#url_obj=repair.genre_to_genre_data(url_obj.document)
#get genre strings
#register the genre with the short genres for faster retrieval
genre_string_list=[]
for g in url_obj.genre:
normalized_string=base_util.normalize_genre_string(g["genre"])
genre_string_list.append(normalized_string)
short_genre_to_genre.select(short_genre=normalized_string).update(upsert=True,add_to_set__genres=g)
Logger.info("Getting bow rep")
#get BOW representation
bow_dict=bow.get_word_count(summarizer.summarize(url_obj.page if isinstance(url_obj.page,str) else base_util.utf_8_safe_decode(url_obj)))
if len(bow_dict)<20:
raise Exception("Words less than 20")
Logger.info("Update count:"+str(bow_dict))
#store the url bow in urlbow table
if not url_to_bow.select(url=url_obj["url"]).find_one():
url_to_bow.create(url=url_obj["url"],bow=bow_dict,short_genres=genre_string_list)
else:
print('Exists bow url number {}'.format(url))
queue.increment()
except Exception as ex:
Logger.error(url_obj['url']+":::"+str(ex),"C:/Users/Kevin/Desktop/GitHub/Research/Webscraper/bad_url_summarize_bow1.txt")
class GetGoodsDetail:
def __init__(self, env='QA'):
self.log = Logger("debug")
opera = OperationIni(fileName='config.ini', pathName='config')
self.env = env
self.get_access_token = GetAccessToken(env=env)
# env字符串转小写
env = env.lower()
key = env + '_url'
self.url = opera.read_ini(section='goods', key=key)
self.path = opera.read_ini(section='goods', key='queryGoodsDetail')
self.access_token = self.get_access_token.get_ini_access_token()
# if env == 'QA':
# self.access_token = opera.read_ini(section='access_token', key='qa_access_token')
# if env == 'DEV':
# self.access_token = opera.read_ini(section='access_token', key='dev_access_token')
def get_goods_detail(self, goodsId, storeId=None):
'''
获取商品详情
:param goodsId: 商品id
:param storeId: 门店id
:return: rsq, 商品skuId
'''
url = self.url.format(self.path, self.access_token)
# json_data = None
if storeId == None:
json_data = {'goodsId': goodsId}
else:
json_data = {'goodsId': goodsId, 'storeId': storeId}
self.log.info('开始：调用get_goods_detail方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
r = requests.post(url=url, json=json_data)
# 如果access_token无效
if r.json()['data'] == 'invalid accesstoken':
# 获取最新的token并存入ini文件
self.log.warning('提示：ini文件中的accesstoken失效，开始获取新的accesstoken')
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
self.log.warning(
'开始：调用get_goods_detail方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
url = self.url.format(self.path, new_access_token)
res = requests.post(url=url, json=json_data)
# print(res.json(), url, json_data)
try:
skuId = res.json()['data']['goods']['skuList'][0]['skuId']
self.log.warning(
'结束：调用get_goods_detail方法，返回数据为:{0}，返回skuId为：{1}'.format(
res.json(), skuId))
return res.json(), skuId
except Exception as f:
# print(f)
self.log.error('调用获取商品详情接口失败，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return res.json()
elif r.json()['code']['errmsg'] == '根据Pid查询storeId失败,此商家不存在此门店':
# print(r.json()['code']['errmsg'])
# return r.json()['code']['errmsg']
# 获取最新的token并存入ini文件
self.log.warning(
'提示：根据Pid查询storeId失败,此商家不存在此门店，尝试开始获取新的accesstoken')
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
url = self.url.format(self.path, new_access_token)
self.log.warning(
'开始：调用get_goods_detail方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
res = requests.post(url=url, json=json_data)
# print(res.json(), url, json_data)
try:
skuId = res.json()['data']['goods']['skuList'][0]['skuId']
self.log.warning(
'结束：调用get_goods_detail方法，返回数据为:{0}，返回skuId为：{1}'.format(
res.json(), skuId))
return res.json(), skuId
except Exception as f:
# print(f)
self.log.error('调用获取商品详情接口失败，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return res.json()
else:
try:
skuId = r.json()['data']['goods']['skuList'][0]['skuId']
self.log.info(
'结束：调用get_goods_detail方法，返回数据为:{0}，返回skuId为：{1}'.format(
r.json(), skuId))
return r.json(), r.json(
)['data']['goods']['skuList'][0]['skuId']
except Exception as f:
# print(f)
self.log.error('调用获取商品详情接口失败1，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return r.json()
class addGoods:
def __init__(self, pid, env='QA'):
self.log = Logger("debug")
opera = OperationIni(fileName='config.ini', pathName='config')
self.env = env
self.pid = pid
self.get_access_token = GetAccessToken(env=env, pid=pid)
# env字符串转小写
env = env.lower()
key = env + '_url'
self.url = opera.read_ini(section='goods', key=key)
self.path = opera.read_ini(section='goods', key='addgoods')
self.access_token = self.get_access_token.get_ini_access_token()
def add_goods(self,
storeId=None,
outerGoodsCode=None,
outerSkuCode=None,
deliveryTypeIdList=None,
title=None,
salePrice=None,
originalPrice=None,
adviseSalePriceMin=None,
adviseSalePriceMax=None,
goodsImageUrl=None):
"""
新增商品
:param storeId: 门店id
:param outerGoodsCode: 外部spu编码
:param outerSkuCode: 商家编码
:param deliveryTypeIdList: 配送类型列表，可传多个配送类型，用,隔开（1.同城限时达;2.全城配;3.包含1和2）
:param title: 商品标题
:param salePrice: 售价
:param originalPrice: 市场价
:param adviseSalePriceMin: 门店售价范围开始值
:param adviseSalePriceMax: 门店售价范围结束值
:param goodsImageUrl: 商品图片
:return:
"""
url = self.url.format(self.path, self.access_token)
pid = self.pid
if self.pid == None:
if self.env == 'QA':
pid = 1
if self.env == 'DEV':
pid = 17
# TODO 预留prod环境
if self.env == 'PROD':
pid = 17
# print(pid)
if storeId == None:
if self.env == "QA":
storeId = 1001
if self.env == "DEV":
storeId = 3017
# TODO 预留prod环境
if self.env == "PROD":
storeId = 3017
if outerGoodsCode == None:
# 使用秒级时间戳自动拼接spu
t = int(time.time())
d = 'spu' + str(t)
outerGoodsCode = d
# 商家编码
if outerSkuCode != None:
outerSkuCode = outerSkuCode
deliveryTypeId = None
if deliveryTypeIdList != None:
if deliveryTypeIdList == '3':
deliveryTypeId = get_delivery_type(
env=self.env,
pid=pid,
storeId=storeId,
deliveryType=int(deliveryTypeIdList))[1]
if deliveryTypeId == None:
return {"status": 103, "message": "当前门店该配送方式不存在"}
elif len(deliveryTypeId) < 2:
return {
"status": 104,
"message": "当前门店只有一种配送方式,请重新传递配送方式ID"
}
else:
deliveryTypeId = deliveryTypeId
else:
deliveryType = get_delivery_type(
env=self.env,
pid=self.pid,
storeId=storeId,
deliveryType=int(deliveryTypeIdList))[1]
if deliveryType == None:
return {"status": 103, "message": "当前门店该配送方式不存在"}
else:
deliveryTypeId = []
deliveryTypeId.append(deliveryType)
if deliveryTypeIdList == None:
if self.env == "QA":
deliveryTypeId = [2]
if self.env == "DEV":
deliveryTypeId = [209435]
# TODO 预留prod环境
if self.env == "PROD":
deliveryTypeId = [209435]
if salePrice == None:
salePrice = 0.01
# if originalPrice == None:
# originalPrice = 1
if adviseSalePriceMin == None:
adviseSalePriceMin = 0.01
if adviseSalePriceMax == None:
adviseSalePriceMax = 1
if goodsImageUrl == None:
goodsImageUrl = "https://image-c.weimobmxd.com/saas-wxbiz/a016cb2de441406289433fd0c71c56bd.png"
json_data = {
"storeId": storeId,
"goods": {
"b2cGoods": {
"deliveryTypeIdList": deliveryTypeId,
"b2cGoodsType": 0
},
"categoryId":
274,
"title":
title,
"isMultiSku":
0,
"outerGoodsCode":
outerGoodsCode,
"goodsTagId":
"",
"goodsDesc":
"",
"goodsTemplateId":
-1,
"isMemberShipDiscount":
0,
"deductStockType":
1,
"isCanSell":
1,
"isAutoCanSell":
0,
"isAutoForbidSell":
0,
"startSellTime":
None,
"startForbidTime":
None,
"categoryNameTree":
"食品,零食/坚果/特产,其他休闲零食",
"skuList": [{
"outerSkuCode": outerSkuCode,
"productType": 1,
"singleProductId": 116130117,
"combineProduct": {},
"salePrice": salePrice,
"adviseSalePriceMin": adviseSalePriceMin,
"adviseSalePriceMax": adviseSalePriceMax,
"originalPrice": originalPrice,
"b2cSku": {
"weight": None,
"volume": None
},
"isDisabled": False,
"editStockNum": 0
}],
"selectedGoodsAttrList": [],
"selectedSaleAttrList": [],
"goodsVideoUrl":
None,
"goodsVideoImageUrl":
None,
"limitBuyNum":
0,
"isPutAway":
0,
"saleChannelType":
3,
"selectedGoodsPropList": [],
"selectedInnerGoodsPropList": [],
"goodsImageUrl": [goodsImageUrl],
"goodsBrandId":
""
}
}
self.log.info('开始：调用add_goods方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
requests.packages.urllib3.disable_warnings()
r = requests.post(url=url, json=json_data, verify=False)
# print(r.json())
# 如果access_token无效
if r.json()['data'] == 'invalid accesstoken':
# 获取最新的token并存入ini文件
self.log.warning('提示：ini文件中的accesstoken失效，开始获取新的accesstoken')
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
url = self.url.format(self.path, new_access_token)
self.log.warning('开始：调用add_goods方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
requests.packages.urllib3.disable_warnings()
res = requests.post(url=url, json=json_data, verify=False)
# print(res.json(), url, json_data)
try:
goodsId = res.json()['data']['goodsId']
skuId = res.json()['data']['skuList'][0]['skuId']
self.log.warning(
'结束：调用add_goods方法，返回数据为:{0}，返回goodsId为：{1}，返回skuId为：{2}'.
format(res.json(), goodsId, skuId))
return res.json(), goodsId, skuId
except Exception as f:
# print(f)
self.log.error('调用新增商品接口失败，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return res.json()
elif r.json()['code']['errmsg'] == '根据Pid查询storeId失败,此商家不存在此门店':
# print(r.json()['code']['errmsg'])
# return r.json()['code']['errmsg']
# 获取最新的token并存入ini文件
self.log.warning(
'提示：根据Pid查询storeId失败,此商家不存在此门店，尝试开始获取新的accesstoken')
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
url = self.url.format(self.path, new_access_token)
self.log.warning('开始：调用add_goods方法，请求地址为：{0}，入参为：{1}'.format(
url, json_data))
requests.packages.urllib3.disable_warnings()
res = requests.post(url=url, json=json_data, verify=False)
# print(res.json(), url, json_data)
try:
goodsId = res.json()['data']['goodsId']
skuId = res.json()['data']['skuList'][0]['skuId']
self.log.warning(
'结束：调用add_goods方法，返回数据为:{0}，返回goodsId为：{1}，返回skuId为：{2}'.
format(res.json(), goodsId, skuId))
return res.json(), goodsId, skuId
except Exception as f:
# print(f)
self.log.error('调用新增商品接口失败，错误日志为：{0}'.format(f))
return {'msg': '根据Pid查询storeId失败,此商家不存在此门店,请检查storeId是否正确'}
else:
try:
goodsId = r.json()['data']['goodsId']
skuId = r.json()['data']['skuList'][0]['skuId']
self.log.warning(
'结束：调用add_goods方法，返回数据为:{0}，返回goodsId为：{1}，返回skuId为：{2}'.
format(r.json(), goodsId, skuId))
return r.json(), goodsId, skuId
except Exception as f:
# print(f)
self.log.error('调用新增商品接口失败1，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return r.json()
class GetOrderDetail:
def __init__(self, pid, env='QA'):
self.log = Logger("debug")
opera = OperationIni(fileName='config.ini', pathName='config')
self.env = env
self.get_access_token = GetAccessToken(env=env, pid=pid)
# env字符串转小写
env = env.lower()
key = env + '_url'
self.url = opera.read_ini(section='goods', key=key)
self.path = opera.read_ini(section='goods', key='queryOrderDetail')
self.access_token = self.get_access_token.get_ini_access_token()
def get_order_detail(self, orderNo):
'''
获取订单详情
:param orderNo: 订单id
:return: rsq
'''
url = self.url.format(self.path, self.access_token)
json_data = {'orderNo': orderNo}
self.log.info('开始：调用get_order_detail方法，请求地址为：{0}，入参为：{1}'.format(url, json_data))
requests.packages.urllib3.disable_warnings()
r = requests.post(url=url, json=json_data, verify=False)
# 如果access_token无效
if r.json()['data'] == 'invalid accesstoken':
# 获取最新的token并存入ini文件
self.log.warning('提示：ini文件中的accesstoken失效，开始获取新的accesstoken')
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
self.log.warning('开始：调用get_order_detail方法，请求地址为：{0}，入参为：{1}'.format(url, json_data))
url = self.url.format(self.path, new_access_token)
requests.packages.urllib3.disable_warnings()
res = requests.post(url=url, json=json_data, verify=False)
# print(res.json(), url, json_data)
try:
self.log.warning('结束：调用get_order_detail方法，返回数据为:{0}'.format(res.json()))
return res.json()
except Exception as f:
# print(f)
self.log.error('调用获取商品详情接口失败，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return res.json()
elif r.json()['code']['errmsg'] == '根据Pid查询storeId失败,此商家不存在此门店':
# print(r.json()['code']['errmsg'])
# return r.json()['code']['errmsg']
# 获取最新的token并存入ini文件
self.log.warning('提示：根据Pid查询storeId失败,此商家不存在此门店，尝试开始获取新的accesstoken')
self.get_access_token.set_access_token()
# 注意：这里一定要重新获取一次ini文件中的access_token
new_access_token = self.get_access_token.get_ini_access_token()
url = self.url.format(self.path, new_access_token)
self.log.warning('开始：调用get_order_detail方法，请求地址为：{0}，入参为：{1}'.format(url, json_data))
requests.packages.urllib3.disable_warnings()
res = requests.post(url=url, json=json_data, verify=False)
# print(res.json(), url, json_data)
try:
self.log.warning('结束：调用get_order_detail方法，返回数据为:{0}'.format(res.json()))
return res.json()
except Exception as f:
# print(f)
self.log.error('调用获取商品详情接口失败，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return res.json()
else:
try:
self.log.info('结束：调用get_order_detail方法，返回数据为:{0}'.format(r.json()))
return r.json()
except Exception as f:
# print(f)
self.log.error('调用获取商品详情接口失败1，错误日志为：{0}'.format(f))
# return {'msg': '底层接口请求失败，请检查所传字段的数据是否正确'}
return r.json()
def get_order_item_id_skuNum(self, orderNo):
'''
获取订单pickingPackageList,storeId,wid
:param orderNo: 订单号
:return: 返回pickingPackageList,storeId,wid
'''
# 调用get_order_detail方法获取返回订单详情数据
result = self.get_order_detail(orderNo=orderNo)
try:
itemList = result['data']['itemList']
# 获取该笔订单下的storeId
storeId = result['data']['merchantInfo']['storeId']
# 获取该笔订单下的wid
wid = result['data']['buyerInfo']['wid']
pickingPackageList = []
for i in itemList:
# 获取itemId
itemId = i['id']
# 获取pickSkuNum
pickSkuNum = i['skuNum']
d = {"itemId": itemId, "pickSkuNum": pickSkuNum}
pickingPackageList.append(d)
return pickingPackageList, storeId, wid
except Exception as f:
self.log.error('获取订单详情中的字段失败，错误日志为：{0}'.format(f))
class BaseModel:
"""Abstract class of model for tensorflow graph"""
AUTHOR = 'demetoir'
def __str__(self):
return "%s_%s" % (self.AUTHOR, self.__class__.__name__)
def __init__(self,
input_shapes=None,
params=None,
logger_path=None,
root_path=ROOT_PATH):
"""create instance of AbstractModel
:type logger_path: str
:param logger_path: path for log file
if logger_path is None, log ony stdout
"""
self.root_path = root_path
if logger_path is None:
self.log = Logger(self.__class__.__name__, LOG_PATH)
else:
self.log = Logger(self.__class__.__name__, logger_path)
self.sess = None
self.saver = None
self.summary_writer = None
self.is_built = False
# gen instance id
self.input_shapes = input_shapes
self.params = params
self.id = "_".join([self.__str__(), time_stamp()])
self.instance_path = os.path.join(INSTANCE_PATH, self.id)
self.instance_visual_result_folder_path = os.path.join(
self.instance_path, VISUAL_RESULT_FOLDER)
self.instance_source_folder_path = os.path.join(
self.instance_path, 'src_code')
self.instance_summary_folder_path = os.path.join(
self.instance_path, 'summary')
self.instance_class_name = self.__class__.__name__
self.instance_source_path = os.path.join(
self.instance_source_folder_path, self.id + '.py')
self.metadata_path = os.path.join(self.instance_path, 'instance.meta')
self.save_folder_path = os.path.join(self.instance_path, 'check_point')
self.check_point_path = os.path.join(self.save_folder_path,
'instance.ckpt')
self.metadata = {
MODEL_METADATA_KEY_INSTANCE_ID: self.id,
MODEL_METADATA_KEY_INSTANCE_PATH: self.instance_path,
MODEL_METADATA_KEY_INSTANCE_VISUAL_RESULT_FOLDER_PATH:
self.instance_visual_result_folder_path,
MODEL_METADATA_KEY_INSTANCE_SOURCE_FOLDER_PATH:
self.instance_source_folder_path,
MODEL_METADATA_KEY_INSTANCE_SOURCE_PATH: self.instance_source_path,
MODEL_METADATA_KEY_INSTANCE_SUMMARY_FOLDER_PATH:
self.instance_summary_folder_path,
MODEL_METADATA_KEY_INSTANCE_CLASS_NAME: self.instance_class_name,
MODEL_METADATA_KEY_METADATA_PATH: self.metadata_path,
MODEL_METADATA_KEY_CHECK_POINT_PATH: self.check_point_path,
MODEL_METADATA_KEY_SAVE_FOLDER_PATH: self.save_folder_path,
MODEL_METADATA_KEY_PARAMS: self.params,
MODEL_METADATA_KEY_INPUT_SHAPES: self.input_shapes,
}
def __del__(self):
# TODO this del need hack
try:
self.close_session()
# reset tensorflow graph
tf.reset_default_graph()
del self.sess
del self.root_path
del self.log
except BaseException as e:
pass
@property
def hyper_param_key(self):
return []
def setup_model(self):
self.log.debug('init directory')
setup_directory(self.instance_path)
setup_directory(self.instance_visual_result_folder_path)
setup_directory(self.instance_source_folder_path)
setup_directory(self.instance_summary_folder_path)
setup_directory(self.save_folder_path)
def load_metadata(self, path):
self.metadata = load_json(path)
self.id = self.metadata[MODEL_METADATA_KEY_INSTANCE_ID]
self.instance_path = self.metadata[MODEL_METADATA_KEY_INSTANCE_PATH]
self.instance_visual_result_folder_path = self.metadata[
MODEL_METADATA_KEY_INSTANCE_VISUAL_RESULT_FOLDER_PATH]
self.instance_source_path = self.metadata[
MODEL_METADATA_KEY_INSTANCE_SOURCE_PATH]
self.instance_class_name = self.metadata[
MODEL_METADATA_KEY_INSTANCE_CLASS_NAME]
self.instance_summary_folder_path = self.metadata[
MODEL_METADATA_KEY_INSTANCE_SUMMARY_FOLDER_PATH]
self.save_folder_path = self.metadata[
MODEL_METADATA_KEY_SAVE_FOLDER_PATH]
self.check_point_path = self.metadata[
MODEL_METADATA_KEY_CHECK_POINT_PATH]
self.params = self.metadata[MODEL_METADATA_KEY_PARAMS]
self.input_shapes = self.metadata[MODEL_METADATA_KEY_INPUT_SHAPES]
def save_metadata(self, path):
self.log.debug('dump metadata')
dump_json(self.metadata, path)
def open_session(self):
if self.sess is None:
self.sess = tf.Session()
self.saver = tf.train.Saver()
self.sess.run(tf.global_variables_initializer())
# self.summary_writer = tf.summary.FileWriter(self.instance_summary_folder_path, self.sess.graph)
else:
raise Exception("fail to open tf session")
def close_session(self):
if self.sess is not None:
self.sess.close()
if self.saver is not None:
self.saver = None
if self.summary_writer is not None:
pass
# self.summary_writer.close()
def build(self):
try:
with tf.variable_scope(str(self.id)):
with tf.variable_scope("misc_ops"):
self.log.debug("build_misc_ops")
self.build_misc_ops()
with tf.variable_scope("hyper_parameter"):
self.log.debug('build_hyper_parameter')
self.hyper_parameter()
self.build_hyper_parameter(self.params)
self.log.debug('build_input_shapes')
if self.input_shapes is None:
raise AttributeError("input_shapes not feed")
self.build_input_shapes(self.input_shapes)
self.log.debug('build_main_graph')
self.build_main_graph()
with tf.variable_scope('loss_function'):
self.log.debug('build_loss_function')
self.build_loss_function()
with tf.variable_scope('train_ops'):
self.log.debug('build_train_ops')
self.build_train_ops()
with tf.variable_scope('summary_ops'):
self.log.debug('build_summary_ops')
self.build_summary_ops()
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.log.error("\n", "".join(traceback.format_tb(exc_traceback)))
raise ModelBuildFailError("ModelBuildFailError")
else:
self.is_built = True
self.log.info("build success")
def build_input_shapes(self, input_shapes):
"""load input shapes for tensor placeholder
:type input_shapes: dict
:param input_shapes: input shapes for tensor placeholder
:raise NotImplementError
if not Implemented
"""
raise NotImplementedError
def build_hyper_parameter(self, params=None):
"""load hyper parameter for model
:param params:
:raise NotImplementError
if not implemented
"""
if params is not None:
for key in self.hyper_param_key:
self.__dict__[key] = params[key]
def build_main_graph(self):
"""load main tensor graph
:raise NotImplementError
if not implemented
"""
raise NotImplementedError
def build_loss_function(self):
"""load loss function of model
:raise NotImplementError
if not implemented
"""
raise NotImplementedError
def build_misc_ops(self):
"""load misc operation of model
:raise NotImplementError
if not implemented
"""
self.global_step = tf.get_variable("global_step",
shape=1,
initializer=tf.zeros_initializer)
self.op_inc_global_step = tf.assign(self.global_step,
self.global_step + 1,
name='op_inc_global_step')
self.global_epoch = tf.get_variable("global_epoch",
shape=1,
initializer=tf.zeros_initializer)
self.op_inc_global_step = tf.assign(self.global_epoch,
self.global_step + 1,
name='op_inc_global_epoch')
def build_train_ops(self):
"""Load train operation of model
:raise NotImplementError
if not implemented
"""
raise NotImplementedError
def build_summary_ops(self):
"""load summary operation for tensorboard
:raise NotImplemented
if not implemented
"""
pass
def write_summary(self,
sess=None,
iter_num=None,
dataset=None,
summary_writer=None):
"""write summary of model for tensorboard
:type sess: Session object for tensorflow.Session
:type iter_num: int
:type dataset: dataset_handler.AbstractDataset
:type summary_writer: tensorflow.summary.FileWriter
:param sess: session object for tensorflow
:param iter_num: current iteration number
:param dataset: dataset for train model
:param summary_writer: file writer for tensorboard summary
:raise NotImplementedError
if not implemented
"""
pass
def hyper_parameter(self):
self.batch_size = None
pass
def save(self):
self.setup_model()
self.save_metadata(self.metadata_path)
if self.sess is None:
self.open_session()
self.saver.save(self.sess, self.check_point_path)
self.log.info("saved at {}".format(self.instance_path))
return self.instance_path
def load(self, path):
path = os.path.join(path, 'instance.meta')
self.load_metadata(path)
self.build()
self.close_session()
self.open_session()
self.saver.restore(self.sess, self.check_point_path)
def get_tf_values(self, fetches, feet_dict):
self.sess.run(fetches, feet_dict)
def if_not_ready_to_train(self):
if not self.is_built:
self.build()
if self.sess is None:
self.open_session()
