def main(rootpath="data/"): logger=Logger(rootpath+"log.txt","ExtAddApp.py",True) configService=ConfigService(rootpath) if not appex.is_running_extension(): print('This script is intended to be run from the sharing extension.') return url = appex.get_url() if not url: console.alert("Error","No input URL found.",'OK', hide_cancel_button=True) if(configService.getLog().getData()==1): logger.error("No input URL found.") return console.hud_alert("正在抓取数据,请等待...","success") appSerVice=AppService(rootpath) res=appSerVice.addApp(url) if(res.equal(ResultEnum.APP_UPDATE)): console.hud_alert("应用更新成功!",'success') elif(res.equal(ResultEnum.SUCCESS)): console.hud_alert("应用添加成功!",'success') else: console.hud_alert(res.getInfo(),'error')
def main(rootpath="data/"): logger = Logger(rootpath + "log.txt", "AutoUpdateApp.py.py", True) configService = ConfigService(rootpath) if (configService.getLog().getData() == 1): logger.info("开始自动更新:") if (not isConnected("http://www.baidu.com")): if (configService.getLog().getData() == 1): logger.error("网络连接超时!\n") return serv = AppService(rootpath) res = serv.updateAllApps() if (not res.equal(ResultEnum.SUCCESS)): if (configService.getLog().getData() == 1): logger.error("自动更新出错: " + res.toString()) else: if (configService.getLog().getData() == 1): logger.info("自动更新完成。\n")
class AppService(object): def __init__(self, rootpath="../data/"): self.rootpath = rootpath dbpath = self.rootpath + "database.db" self.mAppController = AppController(dbpath) self.mPriceService = PriceService(rootpath) self.mConfigService = ConfigService(rootpath) self.mNotification = Notification("AppWishList") self.logger = Logger(self.rootpath + "log.txt", "AppService.py", True) def __del__(self): pass def addApp(self, url, star=True): app = App(url) jsontxt = GetJson(url) if (jsontxt == None): self.logger.error("jsontxt获取失败:" + url) return Result(ResultEnum.URL_INVALID) jsondic = json.loads(jsontxt) if (not IsJsonValid(jsondic)): self.logger.error("无效json:" + url) return Result(ResultEnum.URL_INVALID) app.initByJson(jsondic) if (self.isExist(app)): res = self.updateApp(app) if (res.equal(ResultEnum.SUCCESS)): return Result(ResultEnum.APP_UPDATE) return res self.mAppController.insertApp(app) self.mPriceService.addPrice(app, jsondic) # 默认开启自动更新,并设置为愿望单 app = self.getAppByAppId(app.getAppId()).getData() if (star): self.starApp(app) self.changeAppAutoUpdate(app, 1) else: self.changeAppAutoUpdate(app, 0) # 调用更新时再下载,加快应用添加速度,以免用户长时间等待 # downLoadImage(app.getImgURL(),"data/img/"+app.getAppId()+".png") self.logger.info("添加app:\n" + app.toString() + "\n") return Result(ResultEnum.SUCCESS, app) def getAppByAppId(self, appid): res = self.mAppController.selectAppByAppId(appid) if (res == None): self.logger.error("通过appid查询失败:" + str(appid)) return Result(ResultEnum.SELECT_ERROR) return Result(ResultEnum.SUCCESS, res) def getAllApps(self, where=""): res = self.mAppController.selectAllApps(where) if (res == None): self.logger.error("getAllApps()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) return Result(ResultEnum.SUCCESS, res) def getAppsByCategory(self, category, key=0, desc=0): keys = ["createtime", "name", "price"] descs = ["", "DESC"] order = "ORDER BY " + keys[key] + " " + descs[desc] if (key == 2): res = self.mAppController.selectAppsByCategoryOrderByNewestPrice( category, order) else: res = self.mAppController.selectAppsByCategory(category, order) if (res == None): self.logger.error("getAppsByCategory()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) return Result(ResultEnum.SUCCESS, res) def getAppsByStar(self): res = self.mAppController.selectAppsByStar() if (res == None): self.logger.error("getAppsByStar" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) return Result(ResultEnum.SUCCESS, res) def updateApp(self, app): jsontxt = GetJson(app.getURL()) if (jsontxt == None): self.logger.error("jsontxt获取失败:" + app.getURL()) return Result(ResultEnum.URL_INVALID) jsondic = json.loads(jsontxt) if (not IsJsonValid(jsondic)): self.logger.error("无效json:" + app.getURL()) return Result(ResultEnum.URL_INVALID) app.updateByJson(jsondic) self.mAppController.updateApp(app) self.logger.info("更新app:" + app.getAppId()) res = self.mConfigService.getNotice() if (res.isPositive() and res.getData() == 1): if (not os.path.exists(self.rootpath + "img/" + app.getAppId() + ".png")): downLoadImage(app.getImgURL(), self.rootpath + "img/" + app.getAppId() + ".png") cutImage(self.rootpath + "img/" + app.getAppId() + ".png", (387, 102, 813, 528)) res = self.mPriceService.addPrice(app, jsondic) if (app.getStar() and res.isPositive()): notice = self.mConfigService.getNotice() if (notice.isPositive() and notice.getData() == 1): self.noticePrice(app) return Result(ResultEnum.SUCCESS, app) def updateAppByAppId(self, appid): res = self.getAppByAppId(appid) if (not res.equal(ResultEnum.SUCCESS)): return res else: return self.updateApp(res.getData()) def updateAllApps(self): res = self.getAllApps("WHERE autoupdate=1") if (not res.equal(ResultEnum.SUCCESS)): return res apps = res.getData() newapps = [] fault = 0 for i in apps: res = self.updateApp(i) if (res.isPositive()): newapps.append(res.getData()) else: fault += 1 return Result(ResultEnum.SUCCESS, (newapps, fault)) def updateAllApps_Syn(self, syn=None): res = self.getAllApps("WHERE autoupdate=1") if (not res.equal(ResultEnum.SUCCESS)): return res apps = res.getData() newapps = [] fault = 0 l = len(apps) if (not syn == None): syn(0, l) for i in range(l): if (not syn == None): syn(i + 1, l) res = self.updateApp(apps[i]) if (res.isPositive()): newapps.append(res.getData()) else: fault += 1 return Result(ResultEnum.SUCCESS, (l, fault)) def deleteApp(self, app): if (app.getId() < 0): self.logger.warning("deleteApp()" + ResultEnum.APP_INVALID[1]) return Result(ResultEnum.APP_INVALID) self.mAppController.deleteAppById(app.getId()) self.mPriceService.deletePriceByAppId(app.getAppId()) self.logger.warning("deleteApp()删除app:" + app.getAppId()) return Result(ResultEnum.SUCCESS) def deleteAppByAppId(self, appid): self.mAppController.deleteAppByAppId(appid) self.mPriceService.deletePriceByAppId(appid) self.logger.warning("deleteAppByAppId()删除app:" + appid) return Result(ResultEnum.SUCCESS) def deleteAppsByCategory(self, category): self.mAppController.deleteAppByCategory(category) self.logger.warning("deleteAppByCategory()删除分类为'" + category + "'的app") return Result(ResultEnum.SUCCESS) def deleteAllApps(self): self.mAppController.deleteAllApps() self.logger.warning("删除所有app!") return Result(ResultEnum.SUCCESS) def getCategories(self): res = self.mAppController.selectCategories() if (res == None): self.logger.error("getCategories()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) return Result(ResultEnum.SUCCESS, res) def getPricesByApp(self, app): return self.mPriceService.getPricesByAppId(app.getAppId()) def countApp(self): res = self.mAppController.countApp() return Result(ResultEnum.SUCCESS, res) def countStar(self): res = self.mAppController.countStar() return Result(ResultEnum.SUCCESS, res) def countCategory(self): res = self.mAppController.countCategory() return Result(ResultEnum.SUCCESS, res) def countPrices(self): res = self.mPriceService.countPrices() return Result(ResultEnum.SUCCESS, res.getData()) def sumNewestPrices(self): res = self.mPriceService.sumNewestPrices() return Result(ResultEnum.SUCCESS, res.getData()) def starApp(self, app): cnt = self.sortAppStar().getData() app.setStar(cnt + 1) self.mAppController.updateApp(app) self.logger.info("收藏app:" + app.getAppId()) return Result(ResultEnum.SUCCESS) def unstarApp(self, app): app.setStar(0) self.mAppController.updateApp(app) self.logger.info("取消收藏app:" + app.getAppId()) self.sortAppStar() return Result(ResultEnum.SUCCESS) def changeAppStar(self, app, star): if (star < 1): return self.unstarApp() app.setStar(star) self.mAppController.updateApp(app) self.logger.info("修改收藏app顺序:" + app.getAppId() + " " + str(star)) return Result(ResultEnum.SUCCESS) def sortAppStar(self): res = self.getAppsByStar() if (not res.isPositive()): return res apps = res.getData() cnt = len(apps) for i in range(cnt): self.changeAppStar(apps[i], i + 1) #self.logger.info("排序app收藏顺序。") return Result(ResultEnum.SUCCESS, cnt) def changeAppCategory(self, app, category): self.logger.info("修改app分类: " + str(app.getId()) + " " + app.getApplicationCategory() + " -> " + category) app.setApplicationCategory(category) res = self.mAppController.updateApp(app) return Result(ResultEnum.SUCCESS) def changeAppAutoUpdate(self, app, arg): self.logger.info("修改app自动更新: " + str(app.getAppId()) + " " + str(arg)) app.setAutoUpdate(arg) res = self.mAppController.updateApp(app) return Result(ResultEnum.SUCCESS) def noticePrice(self, app): res = self.getPricesByApp(app) if (not res.isPositive()): return res prices = res.getData() if (len(prices) < 2): return Result(ResultEnum.FAULT) newprice = prices[-1] oldprice = prices[-2] if (newprice.getPrice() >= oldprice.getPrice() or newprice.getNoticed() == 1): return Result(ResultEnum.FAULT) self.mPriceService.setNoticed(newprice) note = '你关注的"' + app.getName() + '"降价啦!🎉 当前价格:¥ ' + str( newprice.getPrice()) self.mNotification.addNotice(note) self.logger.info("降价通知:" + app.getName() + "\n" + newprice.toString()) return Result(ResultEnum.SUCCESS) def isExist(self, app): if (self.mAppController.selectAppByAppId(app.getAppId()) != None): return True else: return False def clearDataBase(self): self.logger.warning("清理数据库:") self.deleteAllApps() self.mPriceService.deleteAllPrices() return Result(ResultEnum.SUCCESS) def setLogger_Run(self, arg): self.logger.setRun(arg) self.mPriceService.setLogger_Run(arg) self.mConfigService.setLogger_Run(arg) return Result(ResultEnum.SUCCESS)
class PriceService(object): def __init__(self, rootpath="data/"): self.rootpath = rootpath dbpath = self.rootpath + "database.db" self.mPriceController = PriceController(dbpath) self.logger = Logger(self.rootpath + "log.txt", "PriceService.py", True) def __del__(self): pass def addPrice(self, app, json): price = Price(app.getAppId(), json["offers"]["price"]) res = self.mPriceController.selectPriceByAppId_Day( app.getAppId(), datetime.datetime.now().strftime('%Y-%m-%d')) if (res == None): self.mPriceController.insertPrice(price) self.logger.info("添加price:" + price.toString()) return Result(ResultEnum.SUCCESS, price) else: res.setPrice(price.getPrice()) self.mPriceController.updatePrice(res) self.logger.info("更新price:" + res.toString()) return Result(ResultEnum.PRICE_UPDATE, res) return Result(ResultEnum.SUCCESS, res) def getPricesByAppId(self, appid): res = self.mPriceController.selectPricesByAppId(appid) if (res == None): self.logger.error("getPricesByAppId()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) return Result(ResultEnum.SUCCESS, res) def setNoticed(self, price): price.setNoticed(1) self.mPriceController.updatePrice(price) self.logger.info("通知price:" + price.toString()) return Result(ResultEnum.PRICE_NOTICED, price) def deletePriceById(self, id): self.mPriceController.deletePriceById(id) self.logger.warning("删除price: id:" + id) return Result(ResultEnum.SUCCESS, id) def deletePriceByAppId(self, appid): self.mPriceController.deletePriceByAppId(appid) self.logger.warning("删除price: appid:" + appid) return Result(ResultEnum.SUCCESS, appid) def deleteAllPrices(self): self.mPriceController.deleteAllPrices() self.logger.warning("删除所有price!") return Result(ResultEnum.SUCCESS) def countPrices(self): res = self.mPriceController.countPrices() return Result(ResultEnum.SUCCESS, res) def sumNewestPrices(self): res = self.mPriceController.sumNewestPrice() return Result(ResultEnum.SUCCESS, res) def setLogger_Run(self, arg): self.logger.setRun(arg)
def create_routes(): bp = Blueprint(__name__, 'nurture') loggre = Logger() try: gpt_t, gpt_m, gpt_a = preLoadGpt2() ctrl_t, ctrl_m, ctrl_a = preLoadCtrl() topic_engine_flag = 1 loggre.info( 'Topic to paragraph generator engine started SUCCESSFULLY!!!') except Exception as e: loggre.error( 'Topic to paragraph generator engine loadedERROR!!!') raise(e) def exception_handler(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except: loggre.error(traceback.format_exc()) return jsonify(code=1201999, msg='inner error'), 200 wrapper.__name__ = func.__name__ return wrapper def post(url, data): headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } return requests.post(url, data=json.dumps(data), headers=headers) @bp.route("/topic_gen", methods=('post', )) @exception_handler def topic_gen(): """ topic to paragraph文本生成 """ body = request.get_json() loggre.info(json.dumps(body)) if 'title' not in body or (not body['title']): loggre.error('title is undefined or it is a invalide value') return jsonify( code=1202002, msg='title is undefined or it is a invalide value.' ) if 'length' not in body or not body['length'] or (str.isdigit(str(body['length'])) != True or 100 > int(body['length']) or int(body['length']) > 500): loggre.error( 'length is undefined or it is a invalide value (length must be an Interger.| length must be between 100 and 500)') return jsonify( code=1202002, msg='length is undefined or it is a invalide value (length must be an Interger.| length must be between 100 and 500)' ), 200 if 'nums' not in body or not body['nums'] or (str.isdigit(str(body['nums'])) != True or 1 > int(body['nums']) or int(body['nums']) > 20): loggre.error( 'nums is undefined or it is a invalide value. (nums must be an Interger.| nums must be between 1 and 20)') return jsonify( code=1202002, msg='nums is undefined or it is a invalide value. (nums must be an Interger.| nums must be between 1 and 20)' ), 200 loggre.info(body) title=str(body['title']) length=int(body['length']) nums=int(body['nums']) if topic_engine_flag == 1: ctrl_sample_nums = int(nums / 1.5) + nums % 2 gpt2_sample_nums = nums - ctrl_sample_nums text_ctrl = genText(ctrl_t, ctrl_m, ctrl_a, length, ctrl_sample_nums, "News " + title) print(text_ctrl) if gpt2_sample_nums != 0: text_gpt2 = genText(gpt_t, gpt_m, gpt_a, length, gpt2_sample_nums, title + " | ") else: text_gpt2 = [] text_list = split_combine_text(text_ctrl, text_gpt2) return jsonify(code=0, msg=0, data=text_list), 200 else: loggre.error( 'Topic to paragraph generator engine is not loaded !') return jsonify( code=1202002, msg='Topic to paragraph generator engine is not loaded !' ), 200 return bp
class ConfigService(object): def __init__(self, rootpath="../data/"): self.rootpath = rootpath dbpath = self.rootpath + "database.db" self.mConfigController = ConfigController(dbpath) self.logger = Logger(self.rootpath + "log.txt", "ConfigService.py", True) def __del__(self): pass def init(self): res = self.mConfigController.selectConfigById(1) if (res == None): self.logger.error("init()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) res.setRunTimes(0) res.setNotice(1) res.setDownLoadImg(1) self.mConfigController.updateConfig(res) self.logger.info("setNotice success") return Result(ResultEnum.SUCCESS, res) def runTimesAddOne(self): res = self.mConfigController.selectConfigById(1) if (res == None): self.logger.error("runTimesAddOne()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) res.setRunTimes(res.getRunTimes() + 1) self.mConfigController.updateConfig(res) return Result(ResultEnum.SUCCESS, res) def setRunTimes(self, times): res = self.mConfigController.selectConfigById(1) if (res == None): self.logger.error("runTimesAddOne()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) res.setRunTimes(times) self.mConfigController.updateConfig(res) return Result(ResultEnum.SUCCESS, res) def setNotice(self, notice): res = self.mConfigController.selectConfigById(1) if (res == None): self.logger.error("setNotice()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) res.setNotice(notice) self.mConfigController.updateConfig(res) self.logger.info("setNotice : " + str(notice)) return Result(ResultEnum.SUCCESS, res) def setDownLoadImg(self, downloadimg): res = self.mConfigController.selectConfigById(1) if (res == None): self.logger.error("setDownLoadImg()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) res.setDownLoadImg(downloadimg) self.mConfigController.updateConfig(res) self.logger.info("setDownLoadImg : " + str(downloadimg)) return Result(ResultEnum.SUCCESS, res) def setLog(self, log): res = self.mConfigController.selectConfigById(1) if (res == None): self.logger.error("setLog()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) res.setLog(log) self.mConfigController.updateConfig(res) self.logger.info("setLog : " + str(log)) return Result(ResultEnum.SUCCESS, res) def setDefult(self): self.logger.info("setDefult:") res = self.setRunTimes(1) if (not res.equal(ResultEnum.SUCCESS)): return res res = self.setNotice(1) if (not res.equal(ResultEnum.SUCCESS)): return res res = self.setDownLoadImg(1) if (not res.equal(ResultEnum.SUCCESS)): return res res = self.setLog(1) if (not res.equal(ResultEnum.SUCCESS)): return res def getConfig(self): res = self.mConfigController.selectConfigById(1) if (res == None): self.logger.error("getConfig()" + ResultEnum.SELECT_ERROR[1]) return Result(ResultEnum.SELECT_ERROR) return Result(ResultEnum.SUCCESS, res) def getRuntimes(self): res = self.getConfig() if (res.isPositive()): return Result(ResultEnum.SUCCESS, res.getData().getRunTimes()) else: self.logger.error("getRuntimes()") return res def getNotice(self): res = self.getConfig() if (res.isPositive()): return Result(ResultEnum.SUCCESS, res.getData().getNotice()) else: self.logger.error("getNotice()") return res def getDownLoadImg(self): res = self.getConfig() if (res.isPositive()): return Result(ResultEnum.SUCCESS, res.getData().getDownLoadImg()) else: self.logger.error("getDownLoadImg()") return res def getLog(self): res = self.getConfig() if (res.isPositive()): return Result(ResultEnum.SUCCESS, res.getData().getLog()) else: self.logger.error("getLog()") return res def setLogger_Run(self, arg): self.logger.setRun(arg)
def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches_string): plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) success_count, failed_count, empty_count = 0, 0, 0 try: searches = json.loads(searches_string) except Exception as e: logger.error('Parsing searches', exception=e, data={'searches': searches_string}) raise e try: service_manager.assert_within_limits(quota=False) geocode_results = geocoder.bulk_geocode(searches) results = [] a_failed_one = None if not geocode_results == EMPTY_BATCH_RESPONSE: for result in geocode_results: metadata = result[2] if len(result) > 2 else {} try: if metadata.get('error', None): results.append([result[0], None, json.dumps(metadata)]) a_failed_one = result failed_count += 1 elif result[1] and len(result[1]) == 2: plan = plpy.prepare( "SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) point = plpy.execute(plan, result[1], 1)[0] results.append([ result[0], point['the_geom'], json.dumps(metadata) ]) success_count += 1 else: results.append([result[0], None, json.dumps(metadata)]) empty_count += 1 except Exception as e: import sys logger.error("Error processing geocode", sys.exc_info(), data={ "username": username, "orgname": orgname }) metadata['processing_error'] = 'Error: {}'.format( e.message) results.append([result[0], None, json.dumps(metadata)]) failed_count += 1 missing_count = len( searches) - success_count - failed_count - empty_count if a_failed_one: logger.warning("failed geocoding", data={ "username": username, "orgname": orgname, "failed": str(a_failed_one), "success_count": success_count, "empty_count": empty_count, "missing_count": missing_count, "failed_count": failed_count }) else: logger.debug("finished geocoding", data={ "username": username, "orgname": orgname, "success_count": success_count, "empty_count": empty_count, "missing_count": missing_count, "failed_count": failed_count }) service_manager.quota_service.increment_success_service_use( success_count) service_manager.quota_service.increment_empty_service_use( empty_count + missing_count) service_manager.quota_service.increment_failed_service_use( failed_count) return results except QuotaExceededException as qe: logger.debug('QuotaExceededException at run_street_point_geocoder', qe, data={ "username": username, "orgname": orgname }) service_manager.quota_service.increment_failed_service_use( len(searches)) return [] except BaseException as e: import sys service_manager.quota_service.increment_failed_service_use( len(searches)) service_manager.logger.error( 'Error trying to bulk geocode street point', sys.exc_info(), data={ "username": username, "orgname": orgname }) raise Exception('Error trying to bulk geocode street') finally: service_manager.quota_service.increment_total_service_use( len(searches))