def load(req):
try:
botid = req.GET.get('botid')
LogService.info('get by session %s ', botid)
a = model.get(botid)
return HttpResponse(a)
except Exception as e:
traceback.print_exc()
return HttpResponse(str(e), status=500)
def drop_all_collections(self):
LogService.info('drop all collections')
status, collection_names = self.client.show_collections()
self.check_status(status)
if len(collection_names) > 0:
for collection_name in collection_names:
status = self.client.drop_collection(collection_name)
self.check_status(status)
LogService.info('%s dropped', collection_name)
def save_vectors(self, vectors, botid):
collection_name = self.decide_collection_name(botid)
self.create_collection_if_need(collection_name)
status, ids = self.client.insert(collection_name=collection_name,
records=vectors)
self.check_status(status)
LogService.info('%d vectors saved into collection %s', len(vectors),
collection_name)
return ids
def clean(self, botid):
if botid is None or len(botid) == 0:
return
if botid in self.model_configs:
model_config = self.model_configs[botid]
model_config.destroy()
del self.model_configs[botid]
LogService.info('model deleted with session %s', botid)
def save(req):
try:
body = req.body.decode('utf-8')
data = json.loads(body)
LogService.debug('save data %s', data)
botid = data['botid']
title = data['title']
a = model.save(botid, title)
return HttpResponse(a)
except Exception as e:
traceback.print_exc()
return HttpResponse(str(e), status=500)
def train(req):
try:
body = req.body.decode('utf-8')
data = json.loads(body)
LogService.debug('train data: %s', data)
questions = data['questions']
answers = data['answers']
botid = model.train(questions, answers)
return HttpResponse(botid)
except Exception as e:
traceback.print_exc()
return HttpResponse(str(e), status=500)
def fit(self):
changed = False
new_model_configs = {}
for (botid, model_config) in self.model_configs.items():
if model_config.invalid():
model_config.destroy()
changed = True
LogService.info('removed invalid bot %s', botid)
else:
new_model_configs[botid] = model_config
if changed:
self.model_configs = new_model_configs
def train(self, questions, answers):
if len(questions) == 0:
raise Exception('请先指定问答')
if len(questions) != len(answers):
raise Exception('问答需一一对应')
botid = '%sv1' % uuid.uuid1().hex
self.model_configs[botid] = ModelConfig.create(self.client, self.embed,
botid, questions,
answers)
LogService.info('model trained with session %s', botid)
return botid
def infer(req):
try:
body = req.body.decode('utf-8')
data = json.loads(body)
LogService.debug('infer data %s', data)
q = data['q']
botid = data['botid']
a = model.infer(q, botid)
return HttpResponse(a)
except Exception as e:
traceback.print_exc()
return HttpResponse(str(e), status=500)
def create_collection_if_need(self, collection_name):
LogService.info('prepare collection %s', collection_name)
status, exists = self.client.has_collection(collection_name)
self.check_status(status)
if not exists:
LogService.info('collection %s not exists, create it.',
collection_name)
create_param = {
'collection_name': collection_name,
'dimension': self.vector_dim,
'index_file_size': 1024,
'metric_type': MetricType.L2
}
status = self.client.create_collection(create_param)
self.check_status(status)
LogService.info('collection %s created', collection_name)
status = self.client.create_index(collection_name,
IndexType.IVF_FLAT,
{'nlist': 16384})
self.check_status(status)
LogService.info('index for collection %s created', collection_name)
else:
LogService.info('collection %s already exists', collection_name)
def search_vectors(self, vectors, botid):
search_param = {'nprobe': 16}
collection_name = self.decide_collection_name(botid)
LogService.info('search vector in collection %s', collection_name)
status, results = self.client.search(collection_name=collection_name,
query_records=vectors,
top_k=1,
params=search_param)
self.check_status(status)
if len(results) > 0:
vid = results[0][0].id
LogService.info('vector found with id %d in collection %s', vid,
collection_name)
return vid
else:
LogService.info('vector not found in collection %s',
collection_name)
return None
def clean(req):
body = req.body.decode('utf-8')
data = json.loads(body)
LogService.debug('clean param: %s', data)
model.clean(data['botid'])
return HttpResponse()
def empty_vectors(self, botid):
collection_name = self.decide_collection_name(botid)
status = self.client.drop_collection(collection_name)
self.check_status(status)
LogService.info('collection %s dropped', collection_name)
def clean_all(self):
self.client.drop_all_collections()
LogService.info('all bots removed')