def eva_a_phi(phi):
na, nnh, nh, nw = phi
# choose a dataset to train (mscoco, flickr8k, flickr30k)
dataset = 'mscoco'
data_dir = osp.join(DATA_ROOT, dataset)
from model.ra import Model
# settings
mb = 64 # mini-batch size
lr = 0.0002 # learning rate
# nh = 512 # size of LSTM's hidden size
# nnh = 512 # hidden size of attention mlp
# nw = 512 # size of word embedding vector
# na = 512 # size of the region features after dimensionality reduction
name = 'ra' # model name, just setting it to 'ra' is ok. 'ra'='region attention'
vocab_freq = 'freq5' # use the vocabulary that filtered out words whose frequences are less than 5
print '... loading data {}'.format(dataset)
train_set = Reader(batch_size=mb, data_split='train', vocab_freq=vocab_freq, stage='train',
data_dir=data_dir, feature_file='features_30res.h5', topic_switch='off') # change 0, 1000, 82783
valid_set = Reader(batch_size=1, data_split='val', vocab_freq=vocab_freq, stage='val',
data_dir=data_dir, feature_file='features_30res.h5',
caption_switch='off', topic_switch='off') # change 0, 10, 5000
npatch, nimg = train_set.features.shape[1:]
nout = len(train_set.vocab)
save_dir = '{}-nnh{}-nh{}-nw{}-na{}-mb{}-V{}'.\
format(dataset.lower(), nnh, nh, nw, na, mb, nout)
save_dir = osp.join(SAVE_ROOT, save_dir)
model_file, m = find_last_snapshot(save_dir, resume_training=False)
os.system('cp model/ra.py {}/'.format(save_dir))
logger = Logger(save_dir)
logger.info('... building')
model = Model(name=name, nimg=nimg, nnh=nnh, nh=nh, na=na, nw=nw, nout=nout, npatch=npatch, model_file=model_file)
# start training
bs = BeamSearch([model], beam_size=1, num_cadidates=100, max_length=20)
best = train(model, bs, train_set, valid_set, save_dir, lr,
display=100, starting=m, endding=20, validation=2000, life=10, logger=logger) # change dis1,100; va 2,2000; life 0,10;
average_models(best=best, L=6, model_dir=save_dir, model_name=name+'.h5') # L 1, 6
# evaluation
np.save('data_dir', data_dir)
np.save('save_dir', save_dir)
os.system('python valid_time.py')
scores = np.load('scores.npy')
running_time = np.load('running_time.npy')
print 'cider:', scores[-1], 'B1-4,C:', scores, 'running time:', running_time
return scores, running_time
def main(rootpath="data/"):
logger = Logger(rootpath + "log.txt", "AutoUpdateApp.py.py", True)
configService = ConfigService(rootpath)
if (configService.getLog().getData() == 1):
logger.info("开始自动更新:")
if (not isConnected("http://www.baidu.com")):
if (configService.getLog().getData() == 1):
logger.error("网络连接超时！\n")
return
serv = AppService(rootpath)
res = serv.updateAllApps()
if (not res.equal(ResultEnum.SUCCESS)):
if (configService.getLog().getData() == 1):
logger.error("自动更新出错: " + res.toString())
else:
if (configService.getLog().getData() == 1):
logger.info("自动更新完成。\n")
def train(model, beam_searcher, train_set, valid_set, save_dir, lr,
display=100, starting=0, endding=20, validation=2000, patience=10, logger=None):
"""
display: output training infomation every 'display' mini-batches
starting: the starting snapshots, > 0 when resuming training
endding: the least training snapshots
validation: evaluate on validation set every 'validation' mini-batches
patience: increase of endding when finds better model
"""
train_func, _ = adam_optimizer(model, lr=lr)
print '... training'
logger = Logger(save_dir) if logger is None else logger
timer = Timer()
loss = 0
imb = starting * validation
best = -1
best_snapshot = -1
timer.tic()
while imb < endding*validation:
imb += 1
x = train_set.iterate_batch()
loss += train_func(*x)[0] / display
if imb % display == 0:
logger.info('snapshot={}, iter={}, loss={:.6f}, time={:.1f} sec'.format(imb/validation, imb, loss, timer.toc()))
timer.tic()
loss = 0
if imb % validation == 0:
saving_index = imb/validation
model.save_to_dir(save_dir, saving_index)
try:
scores = validate(beam_searcher, valid_set, logger)
if scores[3] > best:
best = scores[3]
best_snapshot = saving_index
endding = max(saving_index+patience, endding)
logger.info(' ---- this Bleu-4 = [%.3f], best Bleu-4 = [%.3f], endding -> %d' % \
(scores[3], best, endding))
except OSError:
print '[Ops!! OS Error]'
logger.info('Training done, best snapshot is [%d]' % best_snapshot)
return best_snapshot
def train(model, beam_searcher, train_set, valid_set, save_dir, lr,
display=100, starting=0, endding=20, validation=2000, life=10, logger=None):
"""
display: output training infomation every 'display' mini-batches
starting: the starting snapshots, > 0 when resuming training
endding: the least training snapshots
validation: evaluate on validation set every 'validation' mini-batches
life: increase of endding when finds better model
"""
train_func, _ = adam_optimizer(model, lr=lr)
print '... training'
logger = Logger(save_dir) if logger is None else logger
timer = Timer()
loss = 0
imb = starting * validation
best = -1
best_snapshot = -1
timer.tic()
while imb < endding*validation:
imb += 1
x = train_set.iterate_batch()
loss += train_func(*x)[0] / display
if imb % display == 0:
logger.info('snapshot={}, iter={}, loss={:.6f}, time={:.1f} sec'.format(imb/validation, imb, loss, timer.toc()))
timer.tic()
loss = 0
if imb % validation == 0:
saving_index = imb/validation
model.save_to_dir(save_dir, saving_index)
try:
scores = validate(beam_searcher, valid_set, logger)
if scores[3] > best:
best = scores[3]
best_snapshot = saving_index
endding = max(saving_index+life, endding)
logger.info(' ---- this Bleu-4 = [%.3f], best Bleu-4 = [%.3f], endding -> %d' % \
(scores[3], best, endding))
except OSError:
print '[Ops!! OS Error]'
logger.info('Training done, best snapshot is [%d]' % best_snapshot)
return best_snapshot
for meter in [train_accuracy, test_accuracy, train_loss, test_loss]:
meter.reset()
for index, (data_in, label) in enumerate(train_dataloader):
output = model(data_in)
loss = loss_fn(output, label)
train_loss.update_with_weight(loss.item(), label.shape[0])
optimizer.zero_grad()
loss.backward()
optimizer.step()
top1 = accuracy(output, label)[0]
train_accuracy.update_with_weight(top1, label.shape[0])
model.eval()
for data_in, label in test_dataloader:
output = model(data_in)
loss = loss_fn(output, label)
test_loss.update_with_weight(loss.item(), label.shape[0])
top1 = accuracy(output, label)[0]
test_accuracy.update_with_weight(top1, label.shape[0])
logger.info(" | ".join([
f'Epoch[{epoch}/{training_epochs}]',
f'Train Loss:{train_loss.avg: .3f}',
f'Train Accuracy:{train_accuracy.avg: .3f}%',
f'Test Loss:{test_loss.avg: .3f}',
f'Test Accuracy:{test_accuracy.avg: .3f}%'
]))
class Trainer():
def __init__(self, args):
now_time = datetime.datetime.strftime(datetime.datetime.now(),
'%m%d-%H%M%S')
args.cur_dir = os.path.join(args.exp_dir, now_time)
args.log_path = os.path.join(args.cur_dir, 'train.log')
args.best_model_path = os.path.join(args.cur_dir, 'best_model.pth')
self.args = args
mkdir(self.args.exp_dir)
mkdir(self.args.cur_dir)
self.log = Logger(self.args.log_path, level='debug').logger
self.log.critical("args: \n{}".format(to_str_args(self.args)))
self.train_loader = torch.utils.data.DataLoader(
dataset=CUB200Dataset(root=self.args.root, train=True),
batch_size=self.args.batch_size,
num_workers=self.args.num_workers,
pin_memory=self.args.pin_memory,
shuffle=True)
self.test_loader = torch.utils.data.DataLoader(
dataset=CUB200Dataset(root=self.args.root, train=False),
batch_size=self.args.batch_size,
num_workers=self.args.num_workers,
pin_memory=self.args.pin_memory,
shuffle=False)
self.model = torchvision.models.resnet18(pretrained=True)
self.model.fc = nn.Linear(in_features=self.model.fc.in_features,
out_features=self.args.num_classes)
self.model.cuda()
self.criterion = nn.CrossEntropyLoss()
self.optimizer = torch.optim.Adam(
params=self.model.parameters(), lr=self.args.lr
) if self.args.optim_type == 'Adam' else torch.optim.SGD(
params=self.model.parameters(),
lr=self.args.lr,
momentum=self.args.momentum,
weight_decay=self.args.decay)
self.log.critical("model: \n{}".format(self.model))
self.log.critical("torchsummary: \n{}".format(
summary(model=self.model, input_size=(3, 224, 224))))
self.log.critical("criterion: \n{}".format(self.criterion))
self.log.critical("optimizer: \n{}".format(self.optimizer))
def train(self):
self.model.train()
losses = AverageMeter()
correct = 0
pbar = ImProgressBar(len(self.train_loader))
for i, (imgs, targets) in enumerate(self.train_loader):
imgs, targets = imgs.cuda(), targets.cuda()
outputs = self.model(imgs)
_, predicted = torch.max(outputs.data, dim=1)
correct += (predicted == targets).sum().item()
loss = self.criterion(outputs, targets)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
losses.update(loss.item(), 1)
pbar.update(i)
pbar.finish()
return losses.avg, correct / len(self.train_loader.dataset)
def eval(self, loader):
self.model.eval()
losses = AverageMeter()
correct = 0
with torch.no_grad():
pbar = ImProgressBar(len(loader))
for i, (imgs, targets) in enumerate(loader):
imgs, targets = imgs.cuda(), targets.cuda()
outputs = self.model(imgs)
_, predicted = torch.max(outputs.data, dim=1)
correct += (predicted == targets).sum().item()
loss = self.criterion(outputs, targets)
losses.update(loss.item(), 1)
pbar.update(i)
pbar.finish()
return losses.avg, correct / len(loader.dataset)
def fit(self):
best_epoch, best_test_acc = 0, 0
for epoch in range(0, self.args.epochs):
end = time.time()
train_loss, train_acc = self.train()
test_loss, test_acc = self.eval(self.test_loader)
if test_acc > best_test_acc:
best_epoch = epoch
best_test_acc = test_acc
checkpoint = {
'epoch': epoch + 1,
'args': vars(self.args),
'state_dict': self.model.state_dict(),
'best_test_acc': best_test_acc,
'optimizer': self.optimizer.state_dict()
}
torch.save(checkpoint, self.args.best_model_path)
self.log.info(
'[Epoch: {:3}/{:3}][Time: {:.3f}] Train loss: {:.3f}, Test loss: {:.3f}, Train acc: {:.3f}%, Test acc: {:.3f}% (best_test_acc: {:.3f}%, epoch: {})'
.format(epoch + 1, self.args.epochs,
time.time() - end, train_loss, test_loss,
train_acc * 100, test_acc * 100, best_test_acc * 100,
best_epoch))
class AppService(object):
def __init__(self, rootpath="../data/"):
self.rootpath = rootpath
dbpath = self.rootpath + "database.db"
self.mAppController = AppController(dbpath)
self.mPriceService = PriceService(rootpath)
self.mConfigService = ConfigService(rootpath)
self.mNotification = Notification("AppWishList")
self.logger = Logger(self.rootpath + "log.txt", "AppService.py", True)
def __del__(self):
pass
def addApp(self, url, star=True):
app = App(url)
jsontxt = GetJson(url)
if (jsontxt == None):
self.logger.error("jsontxt获取失败：" + url)
return Result(ResultEnum.URL_INVALID)
jsondic = json.loads(jsontxt)
if (not IsJsonValid(jsondic)):
self.logger.error("无效json：" + url)
return Result(ResultEnum.URL_INVALID)
app.initByJson(jsondic)
if (self.isExist(app)):
res = self.updateApp(app)
if (res.equal(ResultEnum.SUCCESS)):
return Result(ResultEnum.APP_UPDATE)
return res
self.mAppController.insertApp(app)
self.mPriceService.addPrice(app, jsondic)
# 默认开启自动更新，并设置为愿望单
app = self.getAppByAppId(app.getAppId()).getData()
if (star):
self.starApp(app)
self.changeAppAutoUpdate(app, 1)
else:
self.changeAppAutoUpdate(app, 0)
# 调用更新时再下载，加快应用添加速度，以免用户长时间等待
# downLoadImage(app.getImgURL(),"data/img/"+app.getAppId()+".png")
self.logger.info("添加app：\n" + app.toString() + "\n")
return Result(ResultEnum.SUCCESS, app)
def getAppByAppId(self, appid):
res = self.mAppController.selectAppByAppId(appid)
if (res == None):
self.logger.error("通过appid查询失败：" + str(appid))
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getAllApps(self, where=""):
res = self.mAppController.selectAllApps(where)
if (res == None):
self.logger.error("getAllApps()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getAppsByCategory(self, category, key=0, desc=0):
keys = ["createtime", "name", "price"]
descs = ["", "DESC"]
order = "ORDER BY " + keys[key] + " " + descs[desc]
if (key == 2):
res = self.mAppController.selectAppsByCategoryOrderByNewestPrice(
category, order)
else:
res = self.mAppController.selectAppsByCategory(category, order)
if (res == None):
self.logger.error("getAppsByCategory()" +
ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getAppsByStar(self):
res = self.mAppController.selectAppsByStar()
if (res == None):
self.logger.error("getAppsByStar" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def updateApp(self, app):
jsontxt = GetJson(app.getURL())
if (jsontxt == None):
self.logger.error("jsontxt获取失败：" + app.getURL())
return Result(ResultEnum.URL_INVALID)
jsondic = json.loads(jsontxt)
if (not IsJsonValid(jsondic)):
self.logger.error("无效json：" + app.getURL())
return Result(ResultEnum.URL_INVALID)
app.updateByJson(jsondic)
self.mAppController.updateApp(app)
self.logger.info("更新app：" + app.getAppId())
res = self.mConfigService.getNotice()
if (res.isPositive() and res.getData() == 1):
if (not os.path.exists(self.rootpath + "img/" + app.getAppId() +
".png")):
downLoadImage(app.getImgURL(),
self.rootpath + "img/" + app.getAppId() + ".png")
cutImage(self.rootpath + "img/" + app.getAppId() + ".png",
(387, 102, 813, 528))
res = self.mPriceService.addPrice(app, jsondic)
if (app.getStar() and res.isPositive()):
notice = self.mConfigService.getNotice()
if (notice.isPositive() and notice.getData() == 1):
self.noticePrice(app)
return Result(ResultEnum.SUCCESS, app)
def updateAppByAppId(self, appid):
res = self.getAppByAppId(appid)
if (not res.equal(ResultEnum.SUCCESS)):
return res
else:
return self.updateApp(res.getData())
def updateAllApps(self):
res = self.getAllApps("WHERE autoupdate=1")
if (not res.equal(ResultEnum.SUCCESS)):
return res
apps = res.getData()
newapps = []
fault = 0
for i in apps:
res = self.updateApp(i)
if (res.isPositive()):
newapps.append(res.getData())
else:
fault += 1
return Result(ResultEnum.SUCCESS, (newapps, fault))
def updateAllApps_Syn(self, syn=None):
res = self.getAllApps("WHERE autoupdate=1")
if (not res.equal(ResultEnum.SUCCESS)):
return res
apps = res.getData()
newapps = []
fault = 0
l = len(apps)
if (not syn == None):
syn(0, l)
for i in range(l):
if (not syn == None):
syn(i + 1, l)
res = self.updateApp(apps[i])
if (res.isPositive()):
newapps.append(res.getData())
else:
fault += 1
return Result(ResultEnum.SUCCESS, (l, fault))
def deleteApp(self, app):
if (app.getId() < 0):
self.logger.warning("deleteApp()" + ResultEnum.APP_INVALID[1])
return Result(ResultEnum.APP_INVALID)
self.mAppController.deleteAppById(app.getId())
self.mPriceService.deletePriceByAppId(app.getAppId())
self.logger.warning("deleteApp()删除app：" + app.getAppId())
return Result(ResultEnum.SUCCESS)
def deleteAppByAppId(self, appid):
self.mAppController.deleteAppByAppId(appid)
self.mPriceService.deletePriceByAppId(appid)
self.logger.warning("deleteAppByAppId()删除app：" + appid)
return Result(ResultEnum.SUCCESS)
def deleteAppsByCategory(self, category):
self.mAppController.deleteAppByCategory(category)
self.logger.warning("deleteAppByCategory()删除分类为'" + category + "'的app")
return Result(ResultEnum.SUCCESS)
def deleteAllApps(self):
self.mAppController.deleteAllApps()
self.logger.warning("删除所有app!")
return Result(ResultEnum.SUCCESS)
def getCategories(self):
res = self.mAppController.selectCategories()
if (res == None):
self.logger.error("getCategories()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getPricesByApp(self, app):
return self.mPriceService.getPricesByAppId(app.getAppId())
def countApp(self):
res = self.mAppController.countApp()
return Result(ResultEnum.SUCCESS, res)
def countStar(self):
res = self.mAppController.countStar()
return Result(ResultEnum.SUCCESS, res)
def countCategory(self):
res = self.mAppController.countCategory()
return Result(ResultEnum.SUCCESS, res)
def countPrices(self):
res = self.mPriceService.countPrices()
return Result(ResultEnum.SUCCESS, res.getData())
def sumNewestPrices(self):
res = self.mPriceService.sumNewestPrices()
return Result(ResultEnum.SUCCESS, res.getData())
def starApp(self, app):
cnt = self.sortAppStar().getData()
app.setStar(cnt + 1)
self.mAppController.updateApp(app)
self.logger.info("收藏app：" + app.getAppId())
return Result(ResultEnum.SUCCESS)
def unstarApp(self, app):
app.setStar(0)
self.mAppController.updateApp(app)
self.logger.info("取消收藏app：" + app.getAppId())
self.sortAppStar()
return Result(ResultEnum.SUCCESS)
def changeAppStar(self, app, star):
if (star < 1):
return self.unstarApp()
app.setStar(star)
self.mAppController.updateApp(app)
self.logger.info("修改收藏app顺序：" + app.getAppId() + " " + str(star))
return Result(ResultEnum.SUCCESS)
def sortAppStar(self):
res = self.getAppsByStar()
if (not res.isPositive()):
return res
apps = res.getData()
cnt = len(apps)
for i in range(cnt):
self.changeAppStar(apps[i], i + 1)
#self.logger.info("排序app收藏顺序。")
return Result(ResultEnum.SUCCESS, cnt)
def changeAppCategory(self, app, category):
self.logger.info("修改app分类: " + str(app.getId()) + " " +
app.getApplicationCategory() + " -> " + category)
app.setApplicationCategory(category)
res = self.mAppController.updateApp(app)
return Result(ResultEnum.SUCCESS)
def changeAppAutoUpdate(self, app, arg):
self.logger.info("修改app自动更新: " + str(app.getAppId()) + " " + str(arg))
app.setAutoUpdate(arg)
res = self.mAppController.updateApp(app)
return Result(ResultEnum.SUCCESS)
def noticePrice(self, app):
res = self.getPricesByApp(app)
if (not res.isPositive()):
return res
prices = res.getData()
if (len(prices) < 2):
return Result(ResultEnum.FAULT)
newprice = prices[-1]
oldprice = prices[-2]
if (newprice.getPrice() >= oldprice.getPrice()
or newprice.getNoticed() == 1):
return Result(ResultEnum.FAULT)
self.mPriceService.setNoticed(newprice)
note = '你关注的"' + app.getName() + '"降价啦！🎉 当前价格:¥ ' + str(
newprice.getPrice())
self.mNotification.addNotice(note)
self.logger.info("降价通知：" + app.getName() + "\n" + newprice.toString())
return Result(ResultEnum.SUCCESS)
def isExist(self, app):
if (self.mAppController.selectAppByAppId(app.getAppId()) != None):
return True
else:
return False
def clearDataBase(self):
self.logger.warning("清理数据库:")
self.deleteAllApps()
self.mPriceService.deleteAllPrices()
return Result(ResultEnum.SUCCESS)
def setLogger_Run(self, arg):
self.logger.setRun(arg)
self.mPriceService.setLogger_Run(arg)
self.mConfigService.setLogger_Run(arg)
return Result(ResultEnum.SUCCESS)
class PriceService(object):
def __init__(self, rootpath="data/"):
self.rootpath = rootpath
dbpath = self.rootpath + "database.db"
self.mPriceController = PriceController(dbpath)
self.logger = Logger(self.rootpath + "log.txt", "PriceService.py",
True)
def __del__(self):
pass
def addPrice(self, app, json):
price = Price(app.getAppId(), json["offers"]["price"])
res = self.mPriceController.selectPriceByAppId_Day(
app.getAppId(),
datetime.datetime.now().strftime('%Y-%m-%d'))
if (res == None):
self.mPriceController.insertPrice(price)
self.logger.info("添加price：" + price.toString())
return Result(ResultEnum.SUCCESS, price)
else:
res.setPrice(price.getPrice())
self.mPriceController.updatePrice(res)
self.logger.info("更新price：" + res.toString())
return Result(ResultEnum.PRICE_UPDATE, res)
return Result(ResultEnum.SUCCESS, res)
def getPricesByAppId(self, appid):
res = self.mPriceController.selectPricesByAppId(appid)
if (res == None):
self.logger.error("getPricesByAppId()" +
ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def setNoticed(self, price):
price.setNoticed(1)
self.mPriceController.updatePrice(price)
self.logger.info("通知price：" + price.toString())
return Result(ResultEnum.PRICE_NOTICED, price)
def deletePriceById(self, id):
self.mPriceController.deletePriceById(id)
self.logger.warning("删除price: id:" + id)
return Result(ResultEnum.SUCCESS, id)
def deletePriceByAppId(self, appid):
self.mPriceController.deletePriceByAppId(appid)
self.logger.warning("删除price: appid:" + appid)
return Result(ResultEnum.SUCCESS, appid)
def deleteAllPrices(self):
self.mPriceController.deleteAllPrices()
self.logger.warning("删除所有price!")
return Result(ResultEnum.SUCCESS)
def countPrices(self):
res = self.mPriceController.countPrices()
return Result(ResultEnum.SUCCESS, res)
def sumNewestPrices(self):
res = self.mPriceController.sumNewestPrice()
return Result(ResultEnum.SUCCESS, res)
def setLogger_Run(self, arg):
self.logger.setRun(arg)
data_dir=data_dir, feature_file='features_1res.h5',
topic_switch='off')
valid_set = Reader(batch_size=1, data_split='val', vocab_freq=vocab_freq, stage='val',
data_dir=data_dir, feature_file='features_1res.h5',
caption_switch='off', topic_switch='off')
nimg = train_set.features.shape[-1]
nout = len(train_set.vocab)
save_dir = '{}-{}-nh{}-nw{}-mb{}-V{}'.\
format(dataset.lower(), task, nh, nw, mb, nout)
save_dir = osp.join(SAVE_ROOT, save_dir)
model_file, m = find_last_snapshot(save_dir, resume_training=True)
os.system('cp model/gnic.py {}/'.format(save_dir)) # backup the model description
logger = Logger(save_dir)
logger.info('... building')
model = Model(name=name, nimg=nimg, nh=nh, nw=nw, nout=nout, model_file=model_file)
if task == 'ss':
from model.ss import Model
# settings
mb = 64 # mini-batch size
lr = 0.0001 # learning rate
nh = 512 # size of LSTM's hidden size
nw = 512 # size of word embedding vector
name = 'ss' # model name, just setting it to 'sf' is ok. 'sf'='scene factor'
vocab_freq = 'freq5' # use the vocabulary that filtered out words whose frequences are less than 5
print '... loading data {}'.format(dataset)
train_set = Reader(batch_size=mb, data_split='train', vocab_freq=vocab_freq, stage='train',
data_dir=data_dir, feature_file='features_1res.h5', topic_file='lda_topics.h5')
def create_routes():
bp = Blueprint(__name__, 'nurture')
loggre = Logger()
try:
gpt_t, gpt_m, gpt_a = preLoadGpt2()
ctrl_t, ctrl_m, ctrl_a = preLoadCtrl()
topic_engine_flag = 1
loggre.info(
'Topic to paragraph generator engine started SUCCESSFULLY!!!')
except Exception as e:
loggre.error(
'Topic to paragraph generator engine loadedERROR!!!')
raise(e)
def exception_handler(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
loggre.error(traceback.format_exc())
return jsonify(code=1201999, msg='inner error'), 200
wrapper.__name__ = func.__name__
return wrapper
def post(url, data):
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
return requests.post(url, data=json.dumps(data), headers=headers)
@bp.route("/topic_gen", methods=('post', ))
@exception_handler
def topic_gen():
"""
topic to paragraph文本生成
"""
body = request.get_json()
loggre.info(json.dumps(body))
if 'title' not in body or (not body['title']):
loggre.error('title is undefined or it is a invalide value')
return jsonify(
code=1202002,
msg='title is undefined or it is a invalide value.'
)
if 'length' not in body or not body['length'] or (str.isdigit(str(body['length'])) != True
or 100 > int(body['length'])
or int(body['length']) > 500):
loggre.error(
'length is undefined or it is a invalide value (length must be an Interger.| length must be between 100 and 500)')
return jsonify(
code=1202002,
msg='length is undefined or it is a invalide value (length must be an Interger.| length must be between 100 and 500)'
), 200
if 'nums' not in body or not body['nums'] or (str.isdigit(str(body['nums'])) != True
or 1 > int(body['nums'])
or int(body['nums']) > 20):
loggre.error(
'nums is undefined or it is a invalide value. (nums must be an Interger.| nums must be between 1 and 20)')
return jsonify(
code=1202002,
msg='nums is undefined or it is a invalide value. (nums must be an Interger.| nums must be between 1 and 20)'
), 200
loggre.info(body)
title=str(body['title'])
length=int(body['length'])
nums=int(body['nums'])
if topic_engine_flag == 1:
ctrl_sample_nums = int(nums / 1.5) + nums % 2
gpt2_sample_nums = nums - ctrl_sample_nums
text_ctrl = genText(ctrl_t, ctrl_m, ctrl_a, length,
ctrl_sample_nums, "News " + title)
print(text_ctrl)
if gpt2_sample_nums != 0:
text_gpt2 = genText(gpt_t, gpt_m, gpt_a, length,
gpt2_sample_nums, title + " | ")
else:
text_gpt2 = []
text_list = split_combine_text(text_ctrl, text_gpt2)
return jsonify(code=0, msg=0, data=text_list), 200
else:
loggre.error(
'Topic to paragraph generator engine is not loaded !')
return jsonify(
code=1202002,
msg='Topic to paragraph generator engine is not loaded !'
), 200
return bp
class ConfigService(object):
def __init__(self, rootpath="../data/"):
self.rootpath = rootpath
dbpath = self.rootpath + "database.db"
self.mConfigController = ConfigController(dbpath)
self.logger = Logger(self.rootpath + "log.txt", "ConfigService.py",
True)
def __del__(self):
pass
def init(self):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("init()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setRunTimes(0)
res.setNotice(1)
res.setDownLoadImg(1)
self.mConfigController.updateConfig(res)
self.logger.info("setNotice success")
return Result(ResultEnum.SUCCESS, res)
def runTimesAddOne(self):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("runTimesAddOne()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setRunTimes(res.getRunTimes() + 1)
self.mConfigController.updateConfig(res)
return Result(ResultEnum.SUCCESS, res)
def setRunTimes(self, times):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("runTimesAddOne()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setRunTimes(times)
self.mConfigController.updateConfig(res)
return Result(ResultEnum.SUCCESS, res)
def setNotice(self, notice):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("setNotice()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setNotice(notice)
self.mConfigController.updateConfig(res)
self.logger.info("setNotice : " + str(notice))
return Result(ResultEnum.SUCCESS, res)
def setDownLoadImg(self, downloadimg):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("setDownLoadImg()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setDownLoadImg(downloadimg)
self.mConfigController.updateConfig(res)
self.logger.info("setDownLoadImg : " + str(downloadimg))
return Result(ResultEnum.SUCCESS, res)
def setLog(self, log):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("setLog()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
res.setLog(log)
self.mConfigController.updateConfig(res)
self.logger.info("setLog : " + str(log))
return Result(ResultEnum.SUCCESS, res)
def setDefult(self):
self.logger.info("setDefult:")
res = self.setRunTimes(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
res = self.setNotice(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
res = self.setDownLoadImg(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
res = self.setLog(1)
if (not res.equal(ResultEnum.SUCCESS)):
return res
def getConfig(self):
res = self.mConfigController.selectConfigById(1)
if (res == None):
self.logger.error("getConfig()" + ResultEnum.SELECT_ERROR[1])
return Result(ResultEnum.SELECT_ERROR)
return Result(ResultEnum.SUCCESS, res)
def getRuntimes(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getRunTimes())
else:
self.logger.error("getRuntimes()")
return res
def getNotice(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getNotice())
else:
self.logger.error("getNotice()")
return res
def getDownLoadImg(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getDownLoadImg())
else:
self.logger.error("getDownLoadImg()")
return res
def getLog(self):
res = self.getConfig()
if (res.isPositive()):
return Result(ResultEnum.SUCCESS, res.getData().getLog())
else:
self.logger.error("getLog()")
return res
def setLogger_Run(self, arg):
self.logger.setRun(arg)