def test2():
modelidx = [np.random.randint(0,len(t)) for t in ModelIndex]
#gpu_av = GPUtil.getAvailable(order='load', limit=8, maxLoad=0.9, maxMemory=0.9)
#gpu_av = GPUtil.getAvailable(order='random',maxLoad=0.9, maxMemory=0.9, limit=8)
#gpu_av = GPUtil.getAvailable(order='memory', limit=8)
gpu_av = []
gpu_opt = 0
if len(gpu_av)>0:
for i in range(len(gpu_av)):
for j in range(len(GPUs)):
if str(gpu_av[i]) in GPUs[j]:
gpu_opt = 1
modelidx[j] = GPUs[j].index(str(gpu_av[i]))
break
r = request.json
data = r["input"]
quick = False
if "quick" in r:
print("quick pattern")
if r["quick"]=="True":
quick = True
app.logger.info(data)
try:
now = datetime.now()
app.logger.info('time: {}'.format(now))
t0 = time.time()
modl = [model[ii][modelidx[ii]] for ii in range(len(modelidx))]
conf = [config[ii][modelidx[ii]] for ii in range(len(modelidx))]
tokn = [tokenizer[ii][modelidx[ii]] for ii in range(len(modelidx))]
devi = [device[ii][modelidx[ii]] for ii in range(len(modelidx))]
ConfigPredict.gpus = [GPUs[ii][modelidx[ii]] for ii in range(len(modelidx))]
if ConfigPredict.useThread:
result = gpt_gen_thread.generating_thread(app, data, modl, conf, tokn, devi, ConfigPredict,quick, num0,
removeHighFreqWordss=rmHFW, batchGenerating=batchGenerating,tags=tags,
D_simi=D_simi,D_next=D_next,maxNext=maxNext,maxChoice=10)
else:
result = []
for ii in range(len(path_configs)):
gpu = ConfigPredict.gpus[ii]
torch.cuda.set_device(int(gpu))
if ii==1:
r0 = gpt_gen.generating_poem(app,data, model[ii], config[ii], tokenizer[ii],device[ii],quick,num0[ii],batchGenerating=batchGenerating,gpu=gpu)
else:
r0 = gpt_gen.generating(app,data, model[ii], config[ii], tokenizer[ii],device[ii],ConfigPredict,quick=quick,num=num0[ii],removeHighFreqWords=rmHFW[ii],batchGenerating=batchGenerating,gpu=gpu)
r0 = [rr + tags[ii] for rr in r0]
result.extend(r0)
result_nnlm = gpt_gen.nnlm_modelpredict(D_simi,D_next,ConfigPredict,inputStr=data,maxNext=maxNext,maxChoice=10,num=num)
result += [tmp+tags[-1] for tmp in result_nnlm]
t1 = time.time()
modelidx_s = ','.join([str(t) for t in ConfigPredict.gpus])
app.logger.info('gpus {}-th (opt is {}) string and use time: {} s'.format(modelidx_s,gpu_opt,'%0.4f'%(t1-t0)))
#app.logger.info('time for : {}'.format(then - now))
app.logger.info("input:{}".format(data))
app.logger.info("output:\n{}".format('\n'.join(result)))
response = {'message':'success','input':data,'result': result}
except Exception as e:
app.logger.error("error:",e)
response = {'message': 'error', 'input': data, 'result': None}
response_pickled = json.dumps(response)
return Response(response=response_pickled, status=200, mimetype="application/json")
def test2():
r = request.json
data = r["input"]
quick = False
if "quick" in r:
print("quick pattern")
if r["quick"] == "True":
quick = True
app.logger.info(data)
try:
now = datetime.now()
app.logger.info('time: {}'.format(now))
t0 = time.time()
if style == 'poem':
result = gpt_gen.generating_poem(
app,
data,
model,
config,
tokenizer,
device,
quick=quick,
num=num0,
batchGenerating=batchGenerating,
gpu=gpus,
fast_pattern=ConfigPredict.fast_pattern)
else:
result = gpt_gen.generating(app,
data,
model,
config,
tokenizer,
device,
ConfigPredict,
quick=quick,
num=num0,
removeHighFreqWords=rmHFW,
batchGenerating=batchGenerating,
gpu=gpus)
t1 = time.time()
#app.logger.info('time for : {}'.format(then - now))
app.logger.info("input:{}".format(data))
app.logger.info("output:\n{}".format('\n'.join(result)))
app.logger.info("used time:{} s".format('%0.4f' % (t1 - t0)))
response = {'message': 'success', 'input': data, 'result': result}
except Exception as e:
app.logger.error("error:", e)
response = {'message': 'error', 'input': data, 'result': None}
response_pickled = json.dumps(response)
return Response(response=response_pickled,
status=200,
mimetype="application/json")
def test(Data,sym='-new'):
modelidx = [np.random.randint(0, len(t)) for t in ModelIndex]
# gpu_av = GPUtil.getAvailable(order='load', limit=8, maxLoad=0.9, maxMemory=0.9)
# gpu_av = GPUtil.getAvailable(order='random',maxLoad=0.9, maxMemory=0.9, limit=8)
# gpu_av = GPUtil.getAvailable(order='memory', limit=8)
gpu_av = []
gpu_opt = 0
if len(gpu_av) > 0:
for i in range(len(gpu_av)):
for j in range(len(GPUs)):
if str(gpu_av[i]) in GPUs[j]:
gpu_opt = 1
modelidx[j] = GPUs[j].index(str(gpu_av[i]))
break
quick = False
app = ''
t0 = time.time()
modl = [model[ii][modelidx[ii]] for ii in range(len(modelidx))]
conf = [config[ii][modelidx[ii]] for ii in range(len(modelidx))]
tokn = [tokenizer[ii][modelidx[ii]] for ii in range(len(modelidx))]
devi = [device[ii][modelidx[ii]] for ii in range(len(modelidx))]
ConfigPredict.gpus = [GPUs[ii][modelidx[ii]] for ii in range(len(modelidx))]
if ConfigPredict.useThread:
for D in Data:
result = gpt_gen_thread.generating_thread(app, D['input'], modl, conf, tokn, devi, ConfigPredict, quick, num0,
removeHighFreqWordss=rmHFW, batchGenerating=batchGenerating,
tags=tags,
D_simi=D_simi, D_next=D_next, maxNext=maxNext, maxChoice=10)
D['ouput'].extend([r + sym for r in result])
else:
result = []
for ii in range(len(path_configs)):
gpu = ConfigPredict.gpus[ii]
torch.cuda.set_device(int(gpu))
if ii == 1:
r0 = gpt_gen.generating_poem(app, data, model[ii], config[ii], tokenizer[ii], device[ii], quick,
num0[ii], batchGenerating=batchGenerating, gpu=gpu)
else:
r0 = gpt_gen.generating(app, data, model[ii], config[ii], tokenizer[ii], device[ii],
ConfigPredict, quick=quick, num=num0[ii], removeHighFreqWords=rmHFW[ii],
batchGenerating=batchGenerating, gpu=gpu)
r0 = [rr + tags[ii] for rr in r0]
result.extend(r0)
result_nnlm = gpt_gen.nnlm_modelpredict(D_simi, D_next, ConfigPredict, inputStr=data, maxNext=maxNext,
maxChoice=10, num=num)
result += [tmp + tags[-1] for tmp in result_nnlm]
t1 = time.time()
modelidx_s = ','.join([str(t) for t in ConfigPredict.gpus])
print('total inputs:{} and use time: {} s'.format(len(Data), '%0.4f' % (t1 - t0)))
return Data
def test1():
r = request.json
data = r["input"]
quick = False
if "quick" in r:
print("quick pattern")
if r["quick"] == "True":
quick = True
try:
random.shuffle(Model)
model, gpu = Model[0][0], Model[0][1]
if style == 'poem':
result = gpt_gen.generating_poem(
app,
data,
model,
config,
tokenizer,
device,
quick=quick,
num=num0,
batchGenerating=batchGenerating,
gpu=gpu,
fast_pattern=ConfigPredict.fast_pattern)
else:
result = gpt_gen.generating(app,
data,
model,
config,
tokenizer,
device,
ConfigPredict,
quick=quick,
num=num0,
removeHighFreqWords=rmHFW,
batchGenerating=batchGenerating,
gpu=gpu)
response = {'message': 'success', 'input': data, 'result': result}
print('generating success')
except Exception as e:
app.logger.error("error:", e)
response = {'message': 'error', 'input': data, 'result': []}
print('generating failed')
response_pickled = json.dumps(response)
return Response(response=response_pickled,
status=200,
mimetype="application/json")
def main(path_data, mode, path_config, path_target, topk, temp):
ii = int(mode)
model, tokenizer, config, device = gpt_gen.getModel(
path_config=path_config)
config['topk'] = topk
config['temperature'] = temp
with open(path_data, 'r') as f:
s = f.read().strip().split('\n')
D = []
t0 = time.time()
for data in s:
result = []
for _ in range(1):
if ii == 1:
r0 = gpt_gen.generating_poem(app,
data,
model,
config,
tokenizer,
device,
quick,
num0[ii],
batchGenerating=batchGenerating)
else:
r0 = gpt_gen.generating(app,
data,
model,
config,
tokenizer,
device,
ConfigPredict,
quick=quick,
num=num0[ii],
removeHighFreqWords=rmHFW[ii],
batchGenerating=batchGenerating)
r0 = [rr + tags[ii] for rr in r0]
result.extend(r0)
d = {'input': data, 'outputs': result, 'num': len(result)}
D.append(d)
with open(path_target, 'w') as f:
json.dump(D, f, ensure_ascii=False, indent=4)
t1 = time.time()
print('predict time is {} for parameter topk={}'.format(t1 - t0, topk))
def generating_th(self, app, model, prefix, config, tokenizer, device,
ConfigPredict, quick, num, removeHighFreqWords,
batchGenerating):
torch.cuda.set_device(int(self.gpu))
S = generating(app,
prefix,
model,
config,
tokenizer,
device,
ConfigPredict,
quick,
num,
continue_writing=False,
removeHighFreqWords=removeHighFreqWords,
batchGenerating=batchGenerating,
gpu=self.gpu,
onlyMax=self.onlyMax)
return S
def test2():
r = request.json
#print(type(r))
#print(request.json)
#r = '{"input": "们"}'
#r = json.loads(r)
data = r["input"]
if "num" in r:
num = r["num"]
else:
num = 5
quick = False
if "quick" in r:
print("quick pattern")
if r["quick"] == "True":
quick = True
app.logger.info(data)
try:
now = datetime.now()
app.logger.info('time: {}'.format(now))
t0 = time.time()
if ConfigPredict.useThread:
result = gpt_gen_thread.generating_thread(
app,
data,
model,
config,
tokenizer,
device,
ConfigPredict,
quick,
num0,
removeHighFreqWordss=rmHFW,
batchGenerating=batchGenerating,
tags=tags,
D_simi=D_simi,
D_next=D_next,
maxNext=maxNext,
maxChoice=10)
else:
result = []
for ii in range(len(path_configs)):
gpu = ConfigPredict.gpus[ii]
torch.cuda.set_device(int(gpu))
if ii == 1:
r0 = gpt_gen.generating_poem(
app,
data,
model[ii],
config[ii],
tokenizer[ii],
device[ii],
quick,
num0[ii],
batchGenerating=batchGenerating,
gpu=gpu)
else:
r0 = gpt_gen.generating(app,
data,
model[ii],
config[ii],
tokenizer[ii],
device[ii],
ConfigPredict,
quick=quick,
num=num0[ii],
removeHighFreqWords=rmHFW[ii],
batchGenerating=batchGenerating,
gpu=gpu)
r0 = [rr + tags[ii] for rr in r0]
result.extend(r0)
result_nnlm = gpt_gen.nnlm_modelpredict(D_simi,
D_next,
ConfigPredict,
inputStr=data,
maxNext=maxNext,
maxChoice=10,
num=num)
result += [tmp + tags[-1] for tmp in result_nnlm]
t1 = time.time()
app.logger.info('used time: {} s'.format('%0.4f' % (t1 - t0)))
then = datetime.now()
app.logger.info('time: {}'.format(then))
#app.logger.info('time for : {}'.format(then - now))
app.logger.info("input:{}".format(data))
app.logger.info("output:\n{}".format('\n'.join(result)))
response = {'message': 'success', 'input': data, 'result': result}
except Exception as e:
app.logger.error("error:", e)
response = {'message': 'error', 'input': data, 'result': None}
response_pickled = json.dumps(response)
return Response(response=response_pickled,
status=200,
mimetype="application/json")
def main():
S = []
T0 = time.time()
for data in Data:
if ConfigPredict.useThread:
result = gpt_gen_thread.generating_thread(
app,
data,
model,
config,
tokenizer,
device,
ConfigPredict,
quick,
num0,
removeHighFreqWordss=rmHFW,
batchGenerating=batchGenerating,
tags=tags,
D_simi=D_simi,
D_next=D_next,
maxNext=maxNext,
maxChoice=10)
else:
result = []
for ii in range(len(path_configs)):
gpu = ConfigPredict.gpus[ii]
torch.cuda.set_device(int(gpu))
if ii == 1:
r0 = gpt_gen.generating_poem(
app,
data,
model[ii],
config[ii],
tokenizer[ii],
device[ii],
quick,
num0[ii],
batchGenerating=batchGenerating,
gpu=gpu)
else:
r0 = gpt_gen.generating(app,
data,
model[ii],
config[ii],
tokenizer[ii],
device[ii],
ConfigPredict,
quick=quick,
num=num0[ii],
removeHighFreqWords=rmHFW[ii],
batchGenerating=batchGenerating,
gpu=gpu)
r0 = [rr + tags[ii] for rr in r0]
result.extend(r0)
result_nnlm = gpt_gen.nnlm_modelpredict(D_simi,
D_next,
ConfigPredict,
inputStr=data,
maxNext=maxNext,
maxChoice=10,
num=num)
result += [tmp + tags[-1] for tmp in result_nnlm]
response = {'input': data, 'result': result}
S.append(response)
T1 = time.time()
T = T1 - T0
with open(path_target, 'w') as f:
json.dump(S, f, ensure_ascii=False, indent=4)
A = []
for i in range(len(S)):
for j in range(len(S[i]['result'])):
s = S[i]['result'][j]
t = ''.join([s[len(s) - i - 1] for i in range(len(s))])
idx0 = len(s) - t.find('(') - 1
idx1 = len(s) - t.find(')') - 1
tag = s[idx0:idx1 + 1]
if j == 0:
z = [S[i]['input'], S[i]['result'][j]]
else:
z = ['', S[i]['result'][j]]
z.append(tag)
A.append(z)
path_target1 = path_target.replace('json', 'xls')
write_excel(path_target1, A)
print('used time %0.2f and QPS=%0.2f' % (T, len(Data) / T))
