def main(rootpath="data/"):
logger=Logger(rootpath+"log.txt","ExtAddApp.py",True)
configService=ConfigService(rootpath)
if not appex.is_running_extension():
print('This script is intended to be run from the sharing extension.')
return
url = appex.get_url()
if not url:
console.alert("Error","No input URL found.",'OK', hide_cancel_button=True)
if(configService.getLog().getData()==1):
logger.error("No input URL found.")
return
console.hud_alert("正在抓取数据，请等待...","success")
appSerVice=AppService(rootpath)
res=appSerVice.addApp(url)
if(res.equal(ResultEnum.APP_UPDATE)):
console.hud_alert("应用更新成功!",'success')
elif(res.equal(ResultEnum.SUCCESS)):
console.hud_alert("应用添加成功!",'success')
else:
console.hud_alert(res.getInfo(),'error')
def main(rootpath="data/"):
logger = Logger(rootpath + "log.txt", "AutoUpdateApp.py.py", True)
configService = ConfigService(rootpath)
if (configService.getLog().getData() == 1):
logger.info("开始自动更新:")
if (not isConnected("http://www.baidu.com")):
if (configService.getLog().getData() == 1):
logger.error("网络连接超时！\n")
return
serv = AppService(rootpath)
res = serv.updateAllApps()
if (not res.equal(ResultEnum.SUCCESS)):
if (configService.getLog().getData() == 1):
logger.error("自动更新出错: " + res.toString())
else:
if (configService.getLog().getData() == 1):
logger.info("自动更新完成。\n")
class AppService(object):
def __init__(self, rootpath="../data/"):
self.rootpath = rootpath
dbpath = self.rootpath + "database.db"
self.mAppController = AppController(dbpath)
self.mPriceService = PriceService(rootpath)
self.mConfigService = ConfigService(rootpath)
self.mNotification = Notification("AppWishList")
self.logger = Logger(self.rootpath + "log.txt", "AppService.py", True)
def __del__(self):
pass
def addApp(self, url, star=True):
app = App(url)
jsontxt = GetJson(url)
if (jsontxt == None):
self.logger.error("jsontxt获取失败：" + url)
return Result(ResultEnum.URL_INVALID)
jsondic = json.loads(jsontxt)
if (not IsJsonValid(jsondic)):
self.logger.error("无效json：" + url)
return Result(ResultEnum.URL_INVALID)
app.initByJson(jsondic)
if (self.isExist(app)):
res = self.updateApp(app)
if (res.equal(ResultEnum.SUCCESS)):
return Result(ResultEnum.APP_UPDATE)
return res
self.mAppController.insertApp(app)
self.mPriceService.addPrice(app, jsondic)
# 默认开启自动更新，并设置为愿望单
app = self.getAppByAppId(app.getAppId()).getData()
if (star):
self.starApp(app)
self.changeAppAutoUpdate(app, 1)
else:
self.changeAppAutoUpdate(app, 0)
# 调用更新时再下载，加快应用添加速度，以免用户长时间等待
# downLoadImage(app.getImgURL(),"data/img/"+app.getAppId()+".png")
self.logger.info("添加app：\n" + app.toString() + "\n")
return Result(ResultEnum.SUCCESS, app)
def getAppByAppId(self, appid):
res = self.mAppController.selectAppByAppId(appid)
if (res == None):
self.logger.error("通过appid查询失败：" + str(appid))
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getAllApps(self, where=""):
res = self.mAppController.selectAllApps(where)
if (res == None):
self.logger.error("getAllApps()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getAppsByCategory(self, category, key=0, desc=0):
keys = ["createtime", "name", "price"]
descs = ["", "DESC"]
order = "ORDER BY " + keys[key] + " " + descs[desc]
if (key == 2):
res = self.mAppController.selectAppsByCategoryOrderByNewestPrice(
category, order)
else:
res = self.mAppController.selectAppsByCategory(category, order)
if (res == None):
self.logger.error("getAppsByCategory()" +
ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getAppsByStar(self):
res = self.mAppController.selectAppsByStar()
if (res == None):
self.logger.error("getAppsByStar" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def updateApp(self, app):
jsontxt = GetJson(app.getURL())
if (jsontxt == None):
self.logger.error("jsontxt获取失败：" + app.getURL())
return Result(ResultEnum.URL_INVALID)
jsondic = json.loads(jsontxt)
if (not IsJsonValid(jsondic)):
self.logger.error("无效json：" + app.getURL())
return Result(ResultEnum.URL_INVALID)
app.updateByJson(jsondic)
self.mAppController.updateApp(app)
self.logger.info("更新app：" + app.getAppId())
res = self.mConfigService.getNotice()
if (res.isPositive() and res.getData() == 1):
if (not os.path.exists(self.rootpath + "img/" + app.getAppId() +
".png")):
downLoadImage(app.getImgURL(),
self.rootpath + "img/" + app.getAppId() + ".png")
cutImage(self.rootpath + "img/" + app.getAppId() + ".png",
(387, 102, 813, 528))
res = self.mPriceService.addPrice(app, jsondic)
if (app.getStar() and res.isPositive()):
notice = self.mConfigService.getNotice()
if (notice.isPositive() and notice.getData() == 1):
self.noticePrice(app)
return Result(ResultEnum.SUCCESS, app)
def updateAppByAppId(self, appid):
res = self.getAppByAppId(appid)
if (not res.equal(ResultEnum.SUCCESS)):
return res
else:
return self.updateApp(res.getData())
def updateAllApps(self):
res = self.getAllApps("WHERE autoupdate=1")
if (not res.equal(ResultEnum.SUCCESS)):
return res
apps = res.getData()
newapps = []
fault = 0
for i in apps:
res = self.updateApp(i)
if (res.isPositive()):
newapps.append(res.getData())
else:
fault += 1
return Result(ResultEnum.SUCCESS, (newapps, fault))
def updateAllApps_Syn(self, syn=None):
res = self.getAllApps("WHERE autoupdate=1")
if (not res.equal(ResultEnum.SUCCESS)):
return res
apps = res.getData()
newapps = []
fault = 0
l = len(apps)
if (not syn == None):
syn(0, l)
for i in range(l):
if (not syn == None):
syn(i + 1, l)
res = self.updateApp(apps[i])
if (res.isPositive()):
newapps.append(res.getData())
else:
fault += 1
return Result(ResultEnum.SUCCESS, (l, fault))
def deleteApp(self, app):
if (app.getId() < 0):
self.logger.warning("deleteApp()" + ResultEnum.APP_INVALID[1])
return Result(ResultEnum.APP_INVALID)
self.mAppController.deleteAppById(app.getId())
self.mPriceService.deletePriceByAppId(app.getAppId())
self.logger.warning("deleteApp()删除app：" + app.getAppId())
return Result(ResultEnum.SUCCESS)
def deleteAppByAppId(self, appid):
self.mAppController.deleteAppByAppId(appid)
self.mPriceService.deletePriceByAppId(appid)
self.logger.warning("deleteAppByAppId()删除app：" + appid)
return Result(ResultEnum.SUCCESS)
def deleteAppsByCategory(self, category):
self.mAppController.deleteAppByCategory(category)
self.logger.warning("deleteAppByCategory()删除分类为'" + category + "'的app")
return Result(ResultEnum.SUCCESS)
def deleteAllApps(self):
self.mAppController.deleteAllApps()
self.logger.warning("删除所有app!")
return Result(ResultEnum.SUCCESS)
def getCategories(self):
res = self.mAppController.selectCategories()
if (res == None):
self.logger.error("getCategories()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getPricesByApp(self, app):
return self.mPriceService.getPricesByAppId(app.getAppId())
def countApp(self):
res = self.mAppController.countApp()
return Result(ResultEnum.SUCCESS, res)
def countStar(self):
res = self.mAppController.countStar()
return Result(ResultEnum.SUCCESS, res)
def countCategory(self):
res = self.mAppController.countCategory()
return Result(ResultEnum.SUCCESS, res)
def countPrices(self):
res = self.mPriceService.countPrices()
return Result(ResultEnum.SUCCESS, res.getData())
def sumNewestPrices(self):
res = self.mPriceService.sumNewestPrices()
return Result(ResultEnum.SUCCESS, res.getData())
def starApp(self, app):
cnt = self.sortAppStar().getData()
app.setStar(cnt + 1)
self.mAppController.updateApp(app)
self.logger.info("收藏app：" + app.getAppId())
return Result(ResultEnum.SUCCESS)
def unstarApp(self, app):
app.setStar(0)
self.mAppController.updateApp(app)
self.logger.info("取消收藏app：" + app.getAppId())
self.sortAppStar()
return Result(ResultEnum.SUCCESS)
def changeAppStar(self, app, star):
if (star < 1):
return self.unstarApp()
app.setStar(star)
self.mAppController.updateApp(app)
self.logger.info("修改收藏app顺序：" + app.getAppId() + " " + str(star))
return Result(ResultEnum.SUCCESS)
def sortAppStar(self):
res = self.getAppsByStar()
if (not res.isPositive()):
return res
apps = res.getData()
cnt = len(apps)
for i in range(cnt):
self.changeAppStar(apps[i], i + 1)
#self.logger.info("排序app收藏顺序。")
return Result(ResultEnum.SUCCESS, cnt)
def changeAppCategory(self, app, category):
self.logger.info("修改app分类: " + str(app.getId()) + " " +
app.getApplicationCategory() + " -> " + category)
app.setApplicationCategory(category)
res = self.mAppController.updateApp(app)
return Result(ResultEnum.SUCCESS)
def changeAppAutoUpdate(self, app, arg):
self.logger.info("修改app自动更新: " + str(app.getAppId()) + " " + str(arg))
app.setAutoUpdate(arg)
res = self.mAppController.updateApp(app)
return Result(ResultEnum.SUCCESS)
def noticePrice(self, app):
res = self.getPricesByApp(app)
if (not res.isPositive()):
return res
prices = res.getData()
if (len(prices) < 2):
return Result(ResultEnum.FAULT)
newprice = prices[-1]
oldprice = prices[-2]
if (newprice.getPrice() >= oldprice.getPrice()
or newprice.getNoticed() == 1):
return Result(ResultEnum.FAULT)
self.mPriceService.setNoticed(newprice)
note = '你关注的"' + app.getName() + '"降价啦！🎉 当前价格:¥ ' + str(
newprice.getPrice())
self.mNotification.addNotice(note)
self.logger.info("降价通知：" + app.getName() + "\n" + newprice.toString())
return Result(ResultEnum.SUCCESS)
def isExist(self, app):
if (self.mAppController.selectAppByAppId(app.getAppId()) != None):
return True
else:
return False
def clearDataBase(self):
self.logger.warning("清理数据库:")
self.deleteAllApps()
self.mPriceService.deleteAllPrices()
return Result(ResultEnum.SUCCESS)
def setLogger_Run(self, arg):
self.logger.setRun(arg)
self.mPriceService.setLogger_Run(arg)
self.mConfigService.setLogger_Run(arg)
return Result(ResultEnum.SUCCESS)
class PriceService(object):
def __init__(self, rootpath="data/"):
self.rootpath = rootpath
dbpath = self.rootpath + "database.db"
self.mPriceController = PriceController(dbpath)
self.logger = Logger(self.rootpath + "log.txt", "PriceService.py",
True)
def __del__(self):
pass
def addPrice(self, app, json):
price = Price(app.getAppId(), json["offers"]["price"])
res = self.mPriceController.selectPriceByAppId_Day(
app.getAppId(),
datetime.datetime.now().strftime('%Y-%m-%d'))
if (res == None):
self.mPriceController.insertPrice(price)
self.logger.info("添加price：" + price.toString())
return Result(ResultEnum.SUCCESS, price)
else:
res.setPrice(price.getPrice())
self.mPriceController.updatePrice(res)
self.logger.info("更新price：" + res.toString())
return Result(ResultEnum.PRICE_UPDATE, res)
return Result(ResultEnum.SUCCESS, res)
def getPricesByAppId(self, appid):
res = self.mPriceController.selectPricesByAppId(appid)
if (res == None):
self.logger.error("getPricesByAppId()" +
ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def setNoticed(self, price):
price.setNoticed(1)
self.mPriceController.updatePrice(price)
self.logger.info("通知price：" + price.toString())
return Result(ResultEnum.PRICE_NOTICED, price)
def deletePriceById(self, id):
self.mPriceController.deletePriceById(id)
self.logger.warning("删除price: id:" + id)
return Result(ResultEnum.SUCCESS, id)
def deletePriceByAppId(self, appid):
self.mPriceController.deletePriceByAppId(appid)
self.logger.warning("删除price: appid:" + appid)
return Result(ResultEnum.SUCCESS, appid)
def deleteAllPrices(self):
self.mPriceController.deleteAllPrices()
self.logger.warning("删除所有price!")
return Result(ResultEnum.SUCCESS)
def countPrices(self):
res = self.mPriceController.countPrices()
return Result(ResultEnum.SUCCESS, res)
def sumNewestPrices(self):
res = self.mPriceController.sumNewestPrice()
return Result(ResultEnum.SUCCESS, res)
def setLogger_Run(self, arg):
self.logger.setRun(arg)
def create_routes():
bp = Blueprint(__name__, 'nurture')
loggre = Logger()
try:
gpt_t, gpt_m, gpt_a = preLoadGpt2()
ctrl_t, ctrl_m, ctrl_a = preLoadCtrl()
topic_engine_flag = 1
loggre.info(
'Topic to paragraph generator engine started SUCCESSFULLY!!!')
except Exception as e:
loggre.error(
'Topic to paragraph generator engine loadedERROR!!!')
raise(e)
def exception_handler(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
loggre.error(traceback.format_exc())
return jsonify(code=1201999, msg='inner error'), 200
wrapper.__name__ = func.__name__
return wrapper
def post(url, data):
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
return requests.post(url, data=json.dumps(data), headers=headers)
@bp.route("/topic_gen", methods=('post', ))
@exception_handler
def topic_gen():
"""
topic to paragraph文本生成
"""
body = request.get_json()
loggre.info(json.dumps(body))
if 'title' not in body or (not body['title']):
loggre.error('title is undefined or it is a invalide value')
return jsonify(
code=1202002,
msg='title is undefined or it is a invalide value.'
)
if 'length' not in body or not body['length'] or (str.isdigit(str(body['length'])) != True
or 100 > int(body['length'])
or int(body['length']) > 500):
loggre.error(
'length is undefined or it is a invalide value (length must be an Interger.| length must be between 100 and 500)')
return jsonify(
code=1202002,
msg='length is undefined or it is a invalide value (length must be an Interger.| length must be between 100 and 500)'
), 200
if 'nums' not in body or not body['nums'] or (str.isdigit(str(body['nums'])) != True
or 1 > int(body['nums'])
or int(body['nums']) > 20):
loggre.error(
'nums is undefined or it is a invalide value. (nums must be an Interger.| nums must be between 1 and 20)')
return jsonify(
code=1202002,
msg='nums is undefined or it is a invalide value. (nums must be an Interger.| nums must be between 1 and 20)'
), 200
loggre.info(body)
title=str(body['title'])
length=int(body['length'])
nums=int(body['nums'])
if topic_engine_flag == 1:
ctrl_sample_nums = int(nums / 1.5) + nums % 2
gpt2_sample_nums = nums - ctrl_sample_nums
text_ctrl = genText(ctrl_t, ctrl_m, ctrl_a, length,
ctrl_sample_nums, "News " + title)
print(text_ctrl)
if gpt2_sample_nums != 0:
text_gpt2 = genText(gpt_t, gpt_m, gpt_a, length,
gpt2_sample_nums, title + " | ")
else:
text_gpt2 = []
text_list = split_combine_text(text_ctrl, text_gpt2)
return jsonify(code=0, msg=0, data=text_list), 200
else:
loggre.error(
'Topic to paragraph generator engine is not loaded !')
return jsonify(
code=1202002,
msg='Topic to paragraph generator engine is not loaded !'
), 200
return bp
class ConfigService(object):
def __init__(self, rootpath="../data/"):
self.rootpath = rootpath
dbpath = self.rootpath + "database.db"
self.mConfigController = ConfigController(dbpath)
self.logger = Logger(self.rootpath + "log.txt", "ConfigService.py",
True)
def __del__(self):
pass
def init(self):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("init()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setRunTimes(0)
res.setNotice(1)
res.setDownLoadImg(1)
self.mConfigController.updateConfig(res)
self.logger.info("setNotice success")
return Result(ResultEnum.SUCCESS, res)
def runTimesAddOne(self):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("runTimesAddOne()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setRunTimes(res.getRunTimes() + 1)
self.mConfigController.updateConfig(res)
return Result(ResultEnum.SUCCESS, res)
def setRunTimes(self, times):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("runTimesAddOne()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setRunTimes(times)
self.mConfigController.updateConfig(res)
return Result(ResultEnum.SUCCESS, res)
def setNotice(self, notice):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("setNotice()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setNotice(notice)
self.mConfigController.updateConfig(res)
self.logger.info("setNotice : " + str(notice))
return Result(ResultEnum.SUCCESS, res)
def setDownLoadImg(self, downloadimg):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("setDownLoadImg()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setDownLoadImg(downloadimg)
self.mConfigController.updateConfig(res)
self.logger.info("setDownLoadImg : " + str(downloadimg))
return Result(ResultEnum.SUCCESS, res)
def setLog(self, log):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("setLog()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setLog(log)
self.mConfigController.updateConfig(res)
self.logger.info("setLog : " + str(log))
return Result(ResultEnum.SUCCESS, res)
def setDefult(self):
self.logger.info("setDefult:")
res = self.setRunTimes(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
res = self.setNotice(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
res = self.setDownLoadImg(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
res = self.setLog(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
def getConfig(self):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("getConfig()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getRuntimes(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getRunTimes())
else:
self.logger.error("getRuntimes()")
return res
def getNotice(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getNotice())
else:
self.logger.error("getNotice()")
return res
def getDownLoadImg(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getDownLoadImg())
else:
self.logger.error("getDownLoadImg()")
return res
def getLog(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getLog())
else:
self.logger.error("getLog()")
return res
def setLogger_Run(self, arg):
self.logger.setRun(arg)
def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username,
orgname, searches_string):
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
success_count, failed_count, empty_count = 0, 0, 0
try:
searches = json.loads(searches_string)
except Exception as e:
logger.error('Parsing searches',
exception=e,
data={'searches': searches_string})
raise e
try:
service_manager.assert_within_limits(quota=False)
geocode_results = geocoder.bulk_geocode(searches)
results = []
a_failed_one = None
if not geocode_results == EMPTY_BATCH_RESPONSE:
for result in geocode_results:
metadata = result[2] if len(result) > 2 else {}
try:
if metadata.get('error', None):
results.append([result[0], None, json.dumps(metadata)])
a_failed_one = result
failed_count += 1
elif result[1] and len(result[1]) == 2:
plan = plpy.prepare(
"SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ",
["double precision", "double precision"])
point = plpy.execute(plan, result[1], 1)[0]
results.append([
result[0], point['the_geom'],
json.dumps(metadata)
])
success_count += 1
else:
results.append([result[0], None, json.dumps(metadata)])
empty_count += 1
except Exception as e:
import sys
logger.error("Error processing geocode",
sys.exc_info(),
data={
"username": username,
"orgname": orgname
})
metadata['processing_error'] = 'Error: {}'.format(
e.message)
results.append([result[0], None, json.dumps(metadata)])
failed_count += 1
missing_count = len(
searches) - success_count - failed_count - empty_count
if a_failed_one:
logger.warning("failed geocoding",
data={
"username": username,
"orgname": orgname,
"failed": str(a_failed_one),
"success_count": success_count,
"empty_count": empty_count,
"missing_count": missing_count,
"failed_count": failed_count
})
else:
logger.debug("finished geocoding",
data={
"username": username,
"orgname": orgname,
"success_count": success_count,
"empty_count": empty_count,
"missing_count": missing_count,
"failed_count": failed_count
})
service_manager.quota_service.increment_success_service_use(
success_count)
service_manager.quota_service.increment_empty_service_use(
empty_count + missing_count)
service_manager.quota_service.increment_failed_service_use(
failed_count)
return results
except QuotaExceededException as qe:
logger.debug('QuotaExceededException at run_street_point_geocoder',
qe,
data={
"username": username,
"orgname": orgname
})
service_manager.quota_service.increment_failed_service_use(
len(searches))
return []
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use(
len(searches))
service_manager.logger.error(
'Error trying to bulk geocode street point',
sys.exc_info(),
data={
"username": username,
"orgname": orgname
})
raise Exception('Error trying to bulk geocode street')
finally:
service_manager.quota_service.increment_total_service_use(
len(searches))
