def get_genre_refs(*genres): genre_objs=[] for agenre_obj in genres: #find all matching genre if isinstance(agenre_obj,str): genre_models=Genre.objects(genre=agenre_obj) else: genre_models=Genre.objects(genre=agenre_obj['genre']) if(len(genre_models)==0): if not isinstance(agenre_obj,str): genre_model=Genre() for (k,v) in agenre_obj.items(): genre_model[k]=v else: genre_model=Genre(genre=agenre_obj) try: genre_model.save() except: Logger.error("Error saving: "+str(agenre_obj)) genre_objs.append(genre_model) else: genre_objs.extend(genre_models.all()) return genre_objs
def scrape(self,url,parent): Logger.debug('Starting url scrap for {}'.format(url)) config.last_url_and_parent=url+', {}'.format('' if parent==None else parent) new_url=base_util.unreplace_dot_url(url) response=self.http.get(new_url) Logger.debug('Got URL') if not hasattr(response,'data') and new_url.startswith('www.'): new_url=new_url.replace('www.','http://') response=self.http.get(new_url) if not hasattr(response,'data'): new_url=new_url.replace('http://','http://www.') response=self.http.get(new_url) if hasattr(response,'data'): body=base_util.utf_8_safe_decode(response.data) else: Logger.error('No data associated with '+new_url) raise AttributeError(new_url+':::No data') return body,new_url
class GetAuth: def __init__(self, env='QA'): self.log = Logger("debug") self.opera = OperationIni(fileName='config.ini', pathName='config') path = '/website/saas/account/api2/user/login' self.key = env.lower() + '_token' d = get_env_authorization(env=env) self.url = d[0] + path self.cookie = d[1] self.userName = d[2] self.passWord = d[3] # if env == 'QA': # self.url = self.opera.read_ini(section='Authorization', key='qa_url') + path # self.cookie = self.opera.read_ini(section='Authorization', key='qa_cookie') # self.userName = self.opera.read_ini(section='Authorization', key='qa_username') # self.passWord = self.opera.read_ini(section='Authorization', key='qa_password') # if env == 'DEV': # self.url = self.opera.read_ini(section='Authorization', key='dev_url') + path # self.cookie = self.opera.read_ini(section='Authorization', key='dev_cookie') # self.userName = self.opera.read_ini(section='Authorization', key='dev_username') # self.passWord = self.opera.read_ini(section='Authorization', key='dev_password') self.headers = { 'Cookie': self.cookie, 'Content-Type': 'application/x-www-form-urlencoded' } def get_auth(self): ''' 获取token :return: rsp, 登录后的token ''' data = { 'zone': '0086', 'phone': self.userName, 'password': self.passWord, 'remember': False, 'passwordType': 'new' } self.log.info('开始:调用获取B端后台token接口,请求地址为:{0},入参为:{1},请求头为:{2}'.format( self.url, data, self.headers)) r = requests.post(url=self.url, data=data, headers=self.headers) try: token = r.json()['data']['token'] self.log.info('结束:调用获取B端后台token接口,获取到token为:{0}'.format(token)) return token, r.json() except Exception as f: self.log.error('获取B端后台token失败,错误日志为:{0}'.format(f)) print(f) def set_auth(self): ''' 存储最新的后端Authorization :return: ''' token = self.get_auth()[0] self.opera.write_ini(section='Authorization', data=token, key=self.key)
def save_url(**kwargs): kwargs['url']=replace_dot_url(kwargs['url']) url_model=URLToGenre(**kwargs) try: save_obj=url_model.save() except: Logger.error("Error saving: "+str(kwargs['url'])) return save_obj
def push_to_queue(number,url_doc): try: if URLQueue.objects(number=number): return None URLQueue(number=number,document=url_doc).save() except: try: Logger.error('Failed to save with url: {}'.format(url_doc['url'])) except: Logger.error('Complete error to save number: {}'+number)
def __onConsume(self): try: Logger.info("Initializing MQTT Response ...") self.__client = mqtt.Client() self.__client.on_connect = self.__onConnect self.__client.on_message = self.__onMessage broker = self.__properties.get('address.broker') port = int(self.__properties.get('port.broker')) keepAliveBroker = int(self.__properties.get('keep.alive.broker')) subscribe = self.__properties.get('topic.subscribe.broker') self.__client.connect(broker, port, keepAliveBroker) self.__client.subscribe(subscribe) self.__client.loop_forever() except Exception as exception: Logger.error("MQTT Response failed. Cause: " + str(exception))
def scrape_urls_multiproc(cls): #current position pos=MongoDB.get(MetaData,'position',type='queue') #current cap cap=pos process_queue=queue.Queue(maxsize=settings.NUM_PROCESSES) #creates all the necessary processes for p_num in range(0,settings.NUM_PROCESSES): p=mp.Process(target=WebScraper().scrape_links_from_position,args=[cap]) #get curresponding objects process_queue.put(p) cap+=settings.NUM_URLS_PER_PROCESS #now start p.start() head=process_queue.get() #wait and create new processes as needed while(pos<MongoDB.count(URLQueue)): head.join() if not head.exitcode ==0: Logger.error('Error with Process, terminating') return #update counter MongoDB.increment_url_counter(settings.NUM_URLS_PER_PROCESS) p=mp.Process(target=WebScraper().scrape_links_from_position,args=[cap]) process_queue.put(p) p.start() #increase both cap and current position cap+=settings.NUM_URLS_PER_PROCESS pos+=settings.NUM_URLS_PER_PROCESS head=process_queue.get() print(p.exitcode) return cls
def scrape(self): home=self.http.get(dmoz_home) home_page_links=self._scrapeHomeAndGetLinks(home.data) #visit each link in homepage and dig down #for url in home_page_links: i=0 while i<settings.NUM_RANDOM_WEBPAGE: result=self._scrapPage(home_page_links[random.randint(0,len(home_page_links)-1)]) if result is not None and MongoDB.get_url_object(result['url']) is None: i+=1 try: page=utf_8_safe_decode(self.http.get(result['url']).data) MongoDB.save_modify_url(page=page,**result) Logger.info("Completed: "+result['url']) except Exception as ex: Logger.error(ex)
class updateGoodsPrice: def __init__(self, env='QA'): self.log = Logger("debug") self.opera = OperationIni(fileName='config.ini', pathName='config') self.get_skuId = GetGoodsDetail(env=env) self.get_access_token = GetAccessToken(env=env) # env字符串转小写 env = env.lower() key = env + '_url' self.base_url = self.opera.read_ini(section='goods', key=key) self.path = self.opera.read_ini(section='goods', key='updatePrice') self.access_token = self.get_access_token.get_ini_access_token() def update_goods_price(self, storeId, goodsId, originalPrice, salePrice): ''' 修改商品价格 :param storeId: 门店id :param goodsId: 商品id :param originalPrice: 市场价 :param salePrice: 商家统一价 :return: rsq ''' url = self.base_url.format(self.path, self.access_token) # 获取skuId try: self.skuId = self.get_skuId.get_goods_detail(goodsId, storeId)[1] json_data = { 'goodsId': goodsId, 'skuList': [{ 'skuId': self.skuId, 'originalPrice': originalPrice, 'salePrice': salePrice, }], 'storeId': storeId } self.log.info('开始:调用update_goods_price方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) r = requests.post(url=url, json=json_data) # 如果access_token无效 if r.json()['data'] == 'invalid accesstoken': self.log.warning('提示:ini文件中的accesstoken失效,开始获取新的accesstoken') # 获取最新的token并存入ini文件 self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() url = self.base_url.format(self.path, new_access_token) self.log.warning( '开始:调用update_goods_price方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) res = requests.post(url=url, json=json_data) self.log.warning('结束:调用update_goods_price方法,返回数据为:{0}'.format( res.json())) return res.json() else: self.log.info('结束:调用update_goods_price方法,返回数据为:{0}'.format( r.json())) return r.json() except Exception as f: self.log.error('修改商品失败,错误日志为:{0}'.format(f)) return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'}
class MockOrderThrow: def __init__(self, pid, env): self.log = Logger("debug") self.pid = pid self.env = env self.opera = OperationIni() self.get_order_detail = GetOrderDetail(pid=pid, env=env) def mock_order_throw(self, orderNo): url = self.opera.read_ini(self.env, key='mock_order_throw_ip') mock_url = 'http://' + url + ':8080/service' headers = {'Content-Type': 'application/x-www-form-urlencoded'} order_detail = self.get_order_detail.get_order_item_id_skuNum( orderNo=orderNo) pickingPackageList = order_detail[0] storeId = order_detail[1] wid = order_detail[2] # 组装paramterInput参数 paramterInput = [{ "markNo": "111", "orderNo": orderNo, "pickingPackageList": pickingPackageList, "pid": self.pid, "storeId": storeId, "wid": wid }] # 组装参数 data = { 'serviceName': 'orderCenterUpdateExportService', 'methodName': 'pickingAndDelivery', 'paramterInput': '{0}'.format(paramterInput) } # 连接VPN start_vpn() code = None try: self.log.info("开始:调用订单抛出服务接口,请求地址为:{0},入参为:{1},请求头为:{2}".format( mock_url, data, headers)) r = requests.post(url=mock_url, data=data, headers=headers, timeout=3) code = r.status_code result = r.json() print('我要看:{0}'.format(result)) self.log.info("结束:调用订单抛出服务接口,返回数据打印:{0}".format(result)) # 关闭VPN stop_vpn() return result except Exception as f: print(f) status = False # print(status) if status == False or code != 200: self.log.warning('IP已失效,重新获取IP') url = GetSoaServerIp( env=self.env, serviceName='mock_order_throw_servicename').get_soa_url() self.log.warning("获取的新IP为:{0}".format(url)) self.opera.write_ini(section=self.env, data=url, key='mock_order_throw_ip') mock_url = 'http://' + url + ':8080/service' self.log.warning("请求url为:{0},请求data为:{1},请求头为:{2}".format( mock_url, data, headers)) try: self.log.warning( "开始:调用订单抛出服务接口,请求地址为:{0},入参为:{1},请求头为:{2}".format( mock_url, data, headers)) r = requests.post(url=mock_url, data=data, headers=headers) result = r.json() self.log.warning("结束:调用订单抛出服务接口,返回数据打印:{0}".format(result)) # 关闭VPN stop_vpn() return result except Exception as f: msg = {'msg': '发生未知错误,请联系管理员,错误日志为:{0}'.format(f)} self.log.error('发生未知错误,请联系管理员,错误日志为:{0}'.format(f)) # 关闭VPN stop_vpn() return msg # g = MockOrderThrow(pid=1,env='QA') # g.mock_order_throw(orderNo='10094010113')
class updateGoodsStock: def __init__(self, env='QA'): self.log = Logger("debug") opera = OperationIni(fileName='config.ini', pathName='config') self.get_skuId = GetGoodsDetail(env=env) self.get_access_token = GetAccessToken(env=env) # env字符串转小写 env = env.lower() key = env + '_url' self.base_url = opera.read_ini(section='goods', key=key) self.path = opera.read_ini(section='goods', key='wholeUpdateStock') self.access_token = self.get_access_token.get_ini_access_token() # if env == 'QA': # self.access_token = opera.read_ini(section='access_token', key='qa_access_token') # if env == 'DEV': # self.access_token = opera.read_ini(section='access_token', key='dev_access_token') def update_goods_stock(self, goodsId, editStockNum, storeId=None): ''' 修改商品库存 :param goodsId: 商品id :param editStockNum: 需要修改的库存 :param storeId: 门店id :return: rsq ''' url = self.base_url.format(self.path, self.access_token) # 获取skuId try: self.skuId = self.get_skuId.get_goods_detail(goodsId, storeId)[1] json_data = { 'goodsId': goodsId, 'storeId': storeId, 'skuList': [{ 'skuId': self.skuId, 'editStockNum': editStockNum }] } self.log.info('开始:调用update_goods_stock方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) r = requests.post(url=url, json=json_data) # 如果access_token无效 if r.json()['data'] == 'invalid accesstoken': # 获取最新的token并存入ini文件 self.log.warning('提示:ini文件中的accesstoken失效,开始获取新的accesstoken') self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() url = self.base_url.format(self.path, new_access_token) self.log.warning( '开始:调用update_goods_stock方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) res = requests.post(url=url, json=json_data) self.log.warning('结束:调用update_goods_stock方法,返回数据为:{0}'.format( res.json())) return res.json() else: self.log.info('结束:调用update_goods_stock方法,返回数据为:{0}'.format( r.json())) return r.json() except Exception as f: self.log.error('调用获取商品详情接口失败,错误日志为:{0}'.format(f)) return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'}
def collect_bad_url(): """ Make bows of websites in the bad url list :return: """ queue=DBQueue_old("genre_bow") #don't trust anything summarizer=Summarizer() bow=BagOfWords() short_genre_to_genre=coll.ShortGenre() url_to_bow=coll.URLBow() start_pos=queue.get() for c,line in enumerate(open("bad_url_summarize_bow.txt")): if c<start_pos: continue url=line.split(" ")[1].split(":::")[0] try: print('New url {} num: {}'.format(url,c)) url_obj=coll.URLToGenre().select(url=url).find_one() if not hasattr(url_obj,"original") or not url_obj["original"]: print("Not original") continue #request page anyways, most of the bad pages are due to bad pagess data=Request().get_data(base_util.unreplace_dot_url(base_util.unreplace_dot_url(url_obj["url"]))) if data is None: raise Exception('url {} No has page'.format(url)) else: if not hasattr(url_obj,"page") or len(data)>len(url_obj["page"]): print("updating data") data=base_util.utf_8_safe_decode(data) if not hasattr(url_obj,"page"): #save page if the new page is significantly bigger than the old one url_obj.save(page=data) else: url_obj.update(page=data) url_obj.reload() if len(data) > len(url_obj.page): raise Exception("Inconsistency b/w data and page data") #url_obj=repair.genre_to_genre_data(url_obj.document) #get genre strings #register the genre with the short genres for faster retrieval genre_string_list=[] for g in url_obj.genre: normalized_string=base_util.normalize_genre_string(g["genre"]) genre_string_list.append(normalized_string) short_genre_to_genre.select(short_genre=normalized_string).update(upsert=True,add_to_set__genres=g) Logger.info("Getting bow rep") #get BOW representation bow_dict=bow.get_word_count(summarizer.summarize(url_obj.page if isinstance(url_obj.page,str) else base_util.utf_8_safe_decode(url_obj))) if len(bow_dict)<20: raise Exception("Words less than 20") Logger.info("Update count:"+str(bow_dict)) #store the url bow in urlbow table if not url_to_bow.select(url=url_obj["url"]).find_one(): url_to_bow.create(url=url_obj["url"],bow=bow_dict,short_genres=genre_string_list) else: print('Exists bow url number {}'.format(url)) queue.increment() except Exception as ex: Logger.error(url_obj['url']+":::"+str(ex),"C:/Users/Kevin/Desktop/GitHub/Research/Webscraper/bad_url_summarize_bow1.txt")
class GetGoodsDetail: def __init__(self, env='QA'): self.log = Logger("debug") opera = OperationIni(fileName='config.ini', pathName='config') self.env = env self.get_access_token = GetAccessToken(env=env) # env字符串转小写 env = env.lower() key = env + '_url' self.url = opera.read_ini(section='goods', key=key) self.path = opera.read_ini(section='goods', key='queryGoodsDetail') self.access_token = self.get_access_token.get_ini_access_token() # if env == 'QA': # self.access_token = opera.read_ini(section='access_token', key='qa_access_token') # if env == 'DEV': # self.access_token = opera.read_ini(section='access_token', key='dev_access_token') def get_goods_detail(self, goodsId, storeId=None): ''' 获取商品详情 :param goodsId: 商品id :param storeId: 门店id :return: rsq, 商品skuId ''' url = self.url.format(self.path, self.access_token) # json_data = None if storeId == None: json_data = {'goodsId': goodsId} else: json_data = {'goodsId': goodsId, 'storeId': storeId} self.log.info('开始:调用get_goods_detail方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) r = requests.post(url=url, json=json_data) # 如果access_token无效 if r.json()['data'] == 'invalid accesstoken': # 获取最新的token并存入ini文件 self.log.warning('提示:ini文件中的accesstoken失效,开始获取新的accesstoken') self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() self.log.warning( '开始:调用get_goods_detail方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) url = self.url.format(self.path, new_access_token) res = requests.post(url=url, json=json_data) # print(res.json(), url, json_data) try: skuId = res.json()['data']['goods']['skuList'][0]['skuId'] self.log.warning( '结束:调用get_goods_detail方法,返回数据为:{0},返回skuId为:{1}'.format( res.json(), skuId)) return res.json(), skuId except Exception as f: # print(f) self.log.error('调用获取商品详情接口失败,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return res.json() elif r.json()['code']['errmsg'] == '根据Pid查询storeId失败,此商家不存在此门店': # print(r.json()['code']['errmsg']) # return r.json()['code']['errmsg'] # 获取最新的token并存入ini文件 self.log.warning( '提示:根据Pid查询storeId失败,此商家不存在此门店,尝试开始获取新的accesstoken') self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() url = self.url.format(self.path, new_access_token) self.log.warning( '开始:调用get_goods_detail方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) res = requests.post(url=url, json=json_data) # print(res.json(), url, json_data) try: skuId = res.json()['data']['goods']['skuList'][0]['skuId'] self.log.warning( '结束:调用get_goods_detail方法,返回数据为:{0},返回skuId为:{1}'.format( res.json(), skuId)) return res.json(), skuId except Exception as f: # print(f) self.log.error('调用获取商品详情接口失败,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return res.json() else: try: skuId = r.json()['data']['goods']['skuList'][0]['skuId'] self.log.info( '结束:调用get_goods_detail方法,返回数据为:{0},返回skuId为:{1}'.format( r.json(), skuId)) return r.json(), r.json( )['data']['goods']['skuList'][0]['skuId'] except Exception as f: # print(f) self.log.error('调用获取商品详情接口失败1,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return r.json()
class addGoods: def __init__(self, pid, env='QA'): self.log = Logger("debug") opera = OperationIni(fileName='config.ini', pathName='config') self.env = env self.pid = pid self.get_access_token = GetAccessToken(env=env, pid=pid) # env字符串转小写 env = env.lower() key = env + '_url' self.url = opera.read_ini(section='goods', key=key) self.path = opera.read_ini(section='goods', key='addgoods') self.access_token = self.get_access_token.get_ini_access_token() def add_goods(self, storeId=None, outerGoodsCode=None, outerSkuCode=None, deliveryTypeIdList=None, title=None, salePrice=None, originalPrice=None, adviseSalePriceMin=None, adviseSalePriceMax=None, goodsImageUrl=None): """ 新增商品 :param storeId: 门店id :param outerGoodsCode: 外部spu编码 :param outerSkuCode: 商家编码 :param deliveryTypeIdList: 配送类型列表,可传多个配送类型,用,隔开(1.同城限时达;2.全城配;3.包含1和2) :param title: 商品标题 :param salePrice: 售价 :param originalPrice: 市场价 :param adviseSalePriceMin: 门店售价范围开始值 :param adviseSalePriceMax: 门店售价范围结束值 :param goodsImageUrl: 商品图片 :return: """ url = self.url.format(self.path, self.access_token) pid = self.pid if self.pid == None: if self.env == 'QA': pid = 1 if self.env == 'DEV': pid = 17 # TODO 预留prod环境 if self.env == 'PROD': pid = 17 # print(pid) if storeId == None: if self.env == "QA": storeId = 1001 if self.env == "DEV": storeId = 3017 # TODO 预留prod环境 if self.env == "PROD": storeId = 3017 if outerGoodsCode == None: # 使用秒级时间戳自动拼接spu t = int(time.time()) d = 'spu' + str(t) outerGoodsCode = d # 商家编码 if outerSkuCode != None: outerSkuCode = outerSkuCode deliveryTypeId = None if deliveryTypeIdList != None: if deliveryTypeIdList == '3': deliveryTypeId = get_delivery_type( env=self.env, pid=pid, storeId=storeId, deliveryType=int(deliveryTypeIdList))[1] if deliveryTypeId == None: return {"status": 103, "message": "当前门店该配送方式不存在"} elif len(deliveryTypeId) < 2: return { "status": 104, "message": "当前门店只有一种配送方式,请重新传递配送方式ID" } else: deliveryTypeId = deliveryTypeId else: deliveryType = get_delivery_type( env=self.env, pid=self.pid, storeId=storeId, deliveryType=int(deliveryTypeIdList))[1] if deliveryType == None: return {"status": 103, "message": "当前门店该配送方式不存在"} else: deliveryTypeId = [] deliveryTypeId.append(deliveryType) if deliveryTypeIdList == None: if self.env == "QA": deliveryTypeId = [2] if self.env == "DEV": deliveryTypeId = [209435] # TODO 预留prod环境 if self.env == "PROD": deliveryTypeId = [209435] if salePrice == None: salePrice = 0.01 # if originalPrice == None: # originalPrice = 1 if adviseSalePriceMin == None: adviseSalePriceMin = 0.01 if adviseSalePriceMax == None: adviseSalePriceMax = 1 if goodsImageUrl == None: goodsImageUrl = "https://image-c.weimobmxd.com/saas-wxbiz/a016cb2de441406289433fd0c71c56bd.png" json_data = { "storeId": storeId, "goods": { "b2cGoods": { "deliveryTypeIdList": deliveryTypeId, "b2cGoodsType": 0 }, "categoryId": 274, "title": title, "isMultiSku": 0, "outerGoodsCode": outerGoodsCode, "goodsTagId": "", "goodsDesc": "", "goodsTemplateId": -1, "isMemberShipDiscount": 0, "deductStockType": 1, "isCanSell": 1, "isAutoCanSell": 0, "isAutoForbidSell": 0, "startSellTime": None, "startForbidTime": None, "categoryNameTree": "食品,零食/坚果/特产,其他休闲零食", "skuList": [{ "outerSkuCode": outerSkuCode, "productType": 1, "singleProductId": 116130117, "combineProduct": {}, "salePrice": salePrice, "adviseSalePriceMin": adviseSalePriceMin, "adviseSalePriceMax": adviseSalePriceMax, "originalPrice": originalPrice, "b2cSku": { "weight": None, "volume": None }, "isDisabled": False, "editStockNum": 0 }], "selectedGoodsAttrList": [], "selectedSaleAttrList": [], "goodsVideoUrl": None, "goodsVideoImageUrl": None, "limitBuyNum": 0, "isPutAway": 0, "saleChannelType": 3, "selectedGoodsPropList": [], "selectedInnerGoodsPropList": [], "goodsImageUrl": [goodsImageUrl], "goodsBrandId": "" } } self.log.info('开始:调用add_goods方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) requests.packages.urllib3.disable_warnings() r = requests.post(url=url, json=json_data, verify=False) # print(r.json()) # 如果access_token无效 if r.json()['data'] == 'invalid accesstoken': # 获取最新的token并存入ini文件 self.log.warning('提示:ini文件中的accesstoken失效,开始获取新的accesstoken') self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() url = self.url.format(self.path, new_access_token) self.log.warning('开始:调用add_goods方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) requests.packages.urllib3.disable_warnings() res = requests.post(url=url, json=json_data, verify=False) # print(res.json(), url, json_data) try: goodsId = res.json()['data']['goodsId'] skuId = res.json()['data']['skuList'][0]['skuId'] self.log.warning( '结束:调用add_goods方法,返回数据为:{0},返回goodsId为:{1},返回skuId为:{2}'. format(res.json(), goodsId, skuId)) return res.json(), goodsId, skuId except Exception as f: # print(f) self.log.error('调用新增商品接口失败,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return res.json() elif r.json()['code']['errmsg'] == '根据Pid查询storeId失败,此商家不存在此门店': # print(r.json()['code']['errmsg']) # return r.json()['code']['errmsg'] # 获取最新的token并存入ini文件 self.log.warning( '提示:根据Pid查询storeId失败,此商家不存在此门店,尝试开始获取新的accesstoken') self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() url = self.url.format(self.path, new_access_token) self.log.warning('开始:调用add_goods方法,请求地址为:{0},入参为:{1}'.format( url, json_data)) requests.packages.urllib3.disable_warnings() res = requests.post(url=url, json=json_data, verify=False) # print(res.json(), url, json_data) try: goodsId = res.json()['data']['goodsId'] skuId = res.json()['data']['skuList'][0]['skuId'] self.log.warning( '结束:调用add_goods方法,返回数据为:{0},返回goodsId为:{1},返回skuId为:{2}'. format(res.json(), goodsId, skuId)) return res.json(), goodsId, skuId except Exception as f: # print(f) self.log.error('调用新增商品接口失败,错误日志为:{0}'.format(f)) return {'msg': '根据Pid查询storeId失败,此商家不存在此门店,请检查storeId是否正确'} else: try: goodsId = r.json()['data']['goodsId'] skuId = r.json()['data']['skuList'][0]['skuId'] self.log.warning( '结束:调用add_goods方法,返回数据为:{0},返回goodsId为:{1},返回skuId为:{2}'. format(r.json(), goodsId, skuId)) return r.json(), goodsId, skuId except Exception as f: # print(f) self.log.error('调用新增商品接口失败1,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return r.json()
class GetOrderDetail: def __init__(self, pid, env='QA'): self.log = Logger("debug") opera = OperationIni(fileName='config.ini', pathName='config') self.env = env self.get_access_token = GetAccessToken(env=env, pid=pid) # env字符串转小写 env = env.lower() key = env + '_url' self.url = opera.read_ini(section='goods', key=key) self.path = opera.read_ini(section='goods', key='queryOrderDetail') self.access_token = self.get_access_token.get_ini_access_token() def get_order_detail(self, orderNo): ''' 获取订单详情 :param orderNo: 订单id :return: rsq ''' url = self.url.format(self.path, self.access_token) json_data = {'orderNo': orderNo} self.log.info('开始:调用get_order_detail方法,请求地址为:{0},入参为:{1}'.format(url, json_data)) requests.packages.urllib3.disable_warnings() r = requests.post(url=url, json=json_data, verify=False) # 如果access_token无效 if r.json()['data'] == 'invalid accesstoken': # 获取最新的token并存入ini文件 self.log.warning('提示:ini文件中的accesstoken失效,开始获取新的accesstoken') self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() self.log.warning('开始:调用get_order_detail方法,请求地址为:{0},入参为:{1}'.format(url, json_data)) url = self.url.format(self.path, new_access_token) requests.packages.urllib3.disable_warnings() res = requests.post(url=url, json=json_data, verify=False) # print(res.json(), url, json_data) try: self.log.warning('结束:调用get_order_detail方法,返回数据为:{0}'.format(res.json())) return res.json() except Exception as f: # print(f) self.log.error('调用获取商品详情接口失败,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return res.json() elif r.json()['code']['errmsg'] == '根据Pid查询storeId失败,此商家不存在此门店': # print(r.json()['code']['errmsg']) # return r.json()['code']['errmsg'] # 获取最新的token并存入ini文件 self.log.warning('提示:根据Pid查询storeId失败,此商家不存在此门店,尝试开始获取新的accesstoken') self.get_access_token.set_access_token() # 注意:这里一定要重新获取一次ini文件中的access_token new_access_token = self.get_access_token.get_ini_access_token() url = self.url.format(self.path, new_access_token) self.log.warning('开始:调用get_order_detail方法,请求地址为:{0},入参为:{1}'.format(url, json_data)) requests.packages.urllib3.disable_warnings() res = requests.post(url=url, json=json_data, verify=False) # print(res.json(), url, json_data) try: self.log.warning('结束:调用get_order_detail方法,返回数据为:{0}'.format(res.json())) return res.json() except Exception as f: # print(f) self.log.error('调用获取商品详情接口失败,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return res.json() else: try: self.log.info('结束:调用get_order_detail方法,返回数据为:{0}'.format(r.json())) return r.json() except Exception as f: # print(f) self.log.error('调用获取商品详情接口失败1,错误日志为:{0}'.format(f)) # return {'msg': '底层接口请求失败,请检查所传字段的数据是否正确'} return r.json() def get_order_item_id_skuNum(self, orderNo): ''' 获取订单pickingPackageList,storeId,wid :param orderNo: 订单号 :return: 返回pickingPackageList,storeId,wid ''' # 调用get_order_detail方法获取返回订单详情数据 result = self.get_order_detail(orderNo=orderNo) try: itemList = result['data']['itemList'] # 获取该笔订单下的storeId storeId = result['data']['merchantInfo']['storeId'] # 获取该笔订单下的wid wid = result['data']['buyerInfo']['wid'] pickingPackageList = [] for i in itemList: # 获取itemId itemId = i['id'] # 获取pickSkuNum pickSkuNum = i['skuNum'] d = {"itemId": itemId, "pickSkuNum": pickSkuNum} pickingPackageList.append(d) return pickingPackageList, storeId, wid except Exception as f: self.log.error('获取订单详情中的字段失败,错误日志为:{0}'.format(f))
class BaseModel: """Abstract class of model for tensorflow graph""" AUTHOR = 'demetoir' def __str__(self): return "%s_%s" % (self.AUTHOR, self.__class__.__name__) def __init__(self, input_shapes=None, params=None, logger_path=None, root_path=ROOT_PATH): """create instance of AbstractModel :type logger_path: str :param logger_path: path for log file if logger_path is None, log ony stdout """ self.root_path = root_path if logger_path is None: self.log = Logger(self.__class__.__name__, LOG_PATH) else: self.log = Logger(self.__class__.__name__, logger_path) self.sess = None self.saver = None self.summary_writer = None self.is_built = False # gen instance id self.input_shapes = input_shapes self.params = params self.id = "_".join([self.__str__(), time_stamp()]) self.instance_path = os.path.join(INSTANCE_PATH, self.id) self.instance_visual_result_folder_path = os.path.join( self.instance_path, VISUAL_RESULT_FOLDER) self.instance_source_folder_path = os.path.join( self.instance_path, 'src_code') self.instance_summary_folder_path = os.path.join( self.instance_path, 'summary') self.instance_class_name = self.__class__.__name__ self.instance_source_path = os.path.join( self.instance_source_folder_path, self.id + '.py') self.metadata_path = os.path.join(self.instance_path, 'instance.meta') self.save_folder_path = os.path.join(self.instance_path, 'check_point') self.check_point_path = os.path.join(self.save_folder_path, 'instance.ckpt') self.metadata = { MODEL_METADATA_KEY_INSTANCE_ID: self.id, MODEL_METADATA_KEY_INSTANCE_PATH: self.instance_path, MODEL_METADATA_KEY_INSTANCE_VISUAL_RESULT_FOLDER_PATH: self.instance_visual_result_folder_path, MODEL_METADATA_KEY_INSTANCE_SOURCE_FOLDER_PATH: self.instance_source_folder_path, MODEL_METADATA_KEY_INSTANCE_SOURCE_PATH: self.instance_source_path, MODEL_METADATA_KEY_INSTANCE_SUMMARY_FOLDER_PATH: self.instance_summary_folder_path, MODEL_METADATA_KEY_INSTANCE_CLASS_NAME: self.instance_class_name, MODEL_METADATA_KEY_METADATA_PATH: self.metadata_path, MODEL_METADATA_KEY_CHECK_POINT_PATH: self.check_point_path, MODEL_METADATA_KEY_SAVE_FOLDER_PATH: self.save_folder_path, MODEL_METADATA_KEY_PARAMS: self.params, MODEL_METADATA_KEY_INPUT_SHAPES: self.input_shapes, } def __del__(self): # TODO this del need hack try: self.close_session() # reset tensorflow graph tf.reset_default_graph() del self.sess del self.root_path del self.log except BaseException as e: pass @property def hyper_param_key(self): return [] def setup_model(self): self.log.debug('init directory') setup_directory(self.instance_path) setup_directory(self.instance_visual_result_folder_path) setup_directory(self.instance_source_folder_path) setup_directory(self.instance_summary_folder_path) setup_directory(self.save_folder_path) def load_metadata(self, path): self.metadata = load_json(path) self.id = self.metadata[MODEL_METADATA_KEY_INSTANCE_ID] self.instance_path = self.metadata[MODEL_METADATA_KEY_INSTANCE_PATH] self.instance_visual_result_folder_path = self.metadata[ MODEL_METADATA_KEY_INSTANCE_VISUAL_RESULT_FOLDER_PATH] self.instance_source_path = self.metadata[ MODEL_METADATA_KEY_INSTANCE_SOURCE_PATH] self.instance_class_name = self.metadata[ MODEL_METADATA_KEY_INSTANCE_CLASS_NAME] self.instance_summary_folder_path = self.metadata[ MODEL_METADATA_KEY_INSTANCE_SUMMARY_FOLDER_PATH] self.save_folder_path = self.metadata[ MODEL_METADATA_KEY_SAVE_FOLDER_PATH] self.check_point_path = self.metadata[ MODEL_METADATA_KEY_CHECK_POINT_PATH] self.params = self.metadata[MODEL_METADATA_KEY_PARAMS] self.input_shapes = self.metadata[MODEL_METADATA_KEY_INPUT_SHAPES] def save_metadata(self, path): self.log.debug('dump metadata') dump_json(self.metadata, path) def open_session(self): if self.sess is None: self.sess = tf.Session() self.saver = tf.train.Saver() self.sess.run(tf.global_variables_initializer()) # self.summary_writer = tf.summary.FileWriter(self.instance_summary_folder_path, self.sess.graph) else: raise Exception("fail to open tf session") def close_session(self): if self.sess is not None: self.sess.close() if self.saver is not None: self.saver = None if self.summary_writer is not None: pass # self.summary_writer.close() def build(self): try: with tf.variable_scope(str(self.id)): with tf.variable_scope("misc_ops"): self.log.debug("build_misc_ops") self.build_misc_ops() with tf.variable_scope("hyper_parameter"): self.log.debug('build_hyper_parameter') self.hyper_parameter() self.build_hyper_parameter(self.params) self.log.debug('build_input_shapes') if self.input_shapes is None: raise AttributeError("input_shapes not feed") self.build_input_shapes(self.input_shapes) self.log.debug('build_main_graph') self.build_main_graph() with tf.variable_scope('loss_function'): self.log.debug('build_loss_function') self.build_loss_function() with tf.variable_scope('train_ops'): self.log.debug('build_train_ops') self.build_train_ops() with tf.variable_scope('summary_ops'): self.log.debug('build_summary_ops') self.build_summary_ops() except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() self.log.error("\n", "".join(traceback.format_tb(exc_traceback))) raise ModelBuildFailError("ModelBuildFailError") else: self.is_built = True self.log.info("build success") def build_input_shapes(self, input_shapes): """load input shapes for tensor placeholder :type input_shapes: dict :param input_shapes: input shapes for tensor placeholder :raise NotImplementError if not Implemented """ raise NotImplementedError def build_hyper_parameter(self, params=None): """load hyper parameter for model :param params: :raise NotImplementError if not implemented """ if params is not None: for key in self.hyper_param_key: self.__dict__[key] = params[key] def build_main_graph(self): """load main tensor graph :raise NotImplementError if not implemented """ raise NotImplementedError def build_loss_function(self): """load loss function of model :raise NotImplementError if not implemented """ raise NotImplementedError def build_misc_ops(self): """load misc operation of model :raise NotImplementError if not implemented """ self.global_step = tf.get_variable("global_step", shape=1, initializer=tf.zeros_initializer) self.op_inc_global_step = tf.assign(self.global_step, self.global_step + 1, name='op_inc_global_step') self.global_epoch = tf.get_variable("global_epoch", shape=1, initializer=tf.zeros_initializer) self.op_inc_global_step = tf.assign(self.global_epoch, self.global_step + 1, name='op_inc_global_epoch') def build_train_ops(self): """Load train operation of model :raise NotImplementError if not implemented """ raise NotImplementedError def build_summary_ops(self): """load summary operation for tensorboard :raise NotImplemented if not implemented """ pass def write_summary(self, sess=None, iter_num=None, dataset=None, summary_writer=None): """write summary of model for tensorboard :type sess: Session object for tensorflow.Session :type iter_num: int :type dataset: dataset_handler.AbstractDataset :type summary_writer: tensorflow.summary.FileWriter :param sess: session object for tensorflow :param iter_num: current iteration number :param dataset: dataset for train model :param summary_writer: file writer for tensorboard summary :raise NotImplementedError if not implemented """ pass def hyper_parameter(self): self.batch_size = None pass def save(self): self.setup_model() self.save_metadata(self.metadata_path) if self.sess is None: self.open_session() self.saver.save(self.sess, self.check_point_path) self.log.info("saved at {}".format(self.instance_path)) return self.instance_path def load(self, path): path = os.path.join(path, 'instance.meta') self.load_metadata(path) self.build() self.close_session() self.open_session() self.saver.restore(self.sess, self.check_point_path) def get_tf_values(self, fetches, feet_dict): self.sess.run(fetches, feet_dict) def if_not_ready_to_train(self): if not self.is_built: self.build() if self.sess is None: self.open_session()