def classify(self, sentence):
"""
input: sentence
output: result(dictionary)
"""
if self.log is None:
self.log = Logger(self.__class__.__name__,
level=ENV.MODEL_OTHER_LOG_LEVEL.value).logger
sentence = jieba.cut(sentence, cut_all=False)
sentence = ' '.join(sentence)
matrix = self.tfidf.transform([sentence])
self.log.debug(
'In transfered tfidf, the number of words in vocalbulary is: {}'.
format(len(matrix.data)))
result = np.vstack((self.svc.predict_proba(matrix),
self.logistic.predict_proba(matrix),
self.nb.predict_proba(matrix)))
av_pred = np.mean(result, axis=0)
max_pred = np.max(av_pred, axis=0)
max_arg = np.argmax(av_pred)
label = max_arg
label = self.label_mapping[label]
dictionary = {'label': label, 'pred_prob': result, 'av_pred': av_pred}
self.log.debug('Possible labels are: {}'.format(self.label_mapping))
self.log.debug('Other- Final Pred label is: {}'.format(
dictionary['label']))
self.log.debug('Other- svc,logistic,nb result:\n {}'.format(
dictionary['pred_prob']))
self.log.debug('Other- ave result:\n {}'.format(dictionary['av_pred']))
return dictionary
class Classifier_other_base:
def __init__(self, **model):
"""
suggested parameters:
svc, logistic, nb, jieba_path, tfidf
"""
self.log = None
self._load_model(**model)
self._load_attributes(**model)
def _load_model(self, **model):
self.svc = model.get('svc')
self.logistic = model.get('logistic')
self.nb = model.get('nb')
self.tfidf = model.get('tfidf')
# load jieba
jieba_path = model.get('jieba_path')
if jieba_path is not None:
jieba.load_userdict(jieba_path)
def _load_attributes(self, **model):
self.label_mapping = model.get('possible_label')
self.label_mapping = sorted(list(set(self.label_mapping)))
def classify(self, sentence):
"""
input: sentence
output: result(dictionary)
"""
if self.log is None:
self.log = Logger(self.__class__.__name__,
level=ENV.MODEL_OTHER_LOG_LEVEL.value).logger
sentence = jieba.cut(sentence, cut_all=False)
sentence = ' '.join(sentence)
matrix = self.tfidf.transform([sentence])
self.log.debug(
'In transfered tfidf, the number of words in vocalbulary is: {}'.
format(len(matrix.data)))
result = np.vstack((self.svc.predict_proba(matrix),
self.logistic.predict_proba(matrix),
self.nb.predict_proba(matrix)))
av_pred = np.mean(result, axis=0)
max_pred = np.max(av_pred, axis=0)
max_arg = np.argmax(av_pred)
label = max_arg
label = self.label_mapping[label]
dictionary = {'label': label, 'pred_prob': result, 'av_pred': av_pred}
self.log.debug('Possible labels are: {}'.format(self.label_mapping))
self.log.debug('Other- Final Pred label is: {}'.format(
dictionary['label']))
self.log.debug('Other- svc,logistic,nb result:\n {}'.format(
dictionary['pred_prob']))
self.log.debug('Other- ave result:\n {}'.format(dictionary['av_pred']))
return dictionary
def __init__(self, model_dict,max_session=1000,debug=False,host=None,port=None,enableDB=False):
self.max_session = 1000
self.inform_interval = 60
self.inactive_maxlength = 150
#{'uid': {'strategy': Tree(), 'time_response': <time>, 'time_inform': <>}
self.active_session = {}
self.model_dict = model_dict
self.debug=debug
self.enableDB = enableDB
self.log = Logger(self.__class__.__name__,level=ENV.NODE_LOG_LEVEL.value).logger
self.db=DB(host,port,debug,True,enable=self.enableDB)
self._print()
def __init__(self,
node_name,
classifier=None,
msg_path=None,
canJump=False):
self.name = node_name
self._load_message(msg_path)
self.canJump = canJump
self.sentiment = 1
self.sentiment_audit = [self.sentiment]
self.model_name = classifier
self.log = Logger(self.__class__.__name__,
level=ENV.NODE_LOG_LEVEL.value).logger
def __init__(self,
graph_path,
msg_path,
model_dict,
max_session=1000,
debug=False):
self.max_session = 1000
self.inform_interval = 60
self.inactive_maxlength = 150
#{'uid': {'strategy': Tree(), 'time_response': <time>, 'time_inform': <>}
self.active_session = {}
self.model_dict = model_dict
self.graph_path = graph_path
self.msg_path = msg_path
self.debug = debug
self.log = Logger(self.__class__.__name__,
level=ENV.NODE_LOG_LEVEL.value).logger
self._print()
self.mongo_db = MongoClient('localhost', 27017).chatbot
def __init__(self, profile=None):
"""
profile should be None or dictionary:
fields:
1. name: lastName + firstName, eg "Li Ming"
if name is None, the constructor will try to load "lastName" and "firstName"
2. principal: the money borrowed, eg:'10,000'
3. contractStartDate, the date when money was borrowed. eg:"2018年5月2日", format"dddd年dd月dd日"
4. contractEndDate, the date before when total amount should be paid.
eg:"2018年5月2日", format"dddd年dd月dd日"
5. apr: yearly/monthly, no calculation will be involved. type: string. eg, '9%'
6. fee: late payment fee. string, eg "500"
7. lendingCompany: the money originally borrowed from
type, string, eg "平安E贷"
8. collectionCompany
type, string, eg "江苏逸能"
9. customerID
string or int "100000"
10. gender
string, "男/女"
11. collector: the agent who makes the call
string : "李明"
12. totalAmount: the total amount owed by debotor
string: “50,000”
13. informDeadline: the deadline to collect money
相对时间
string: “明天下午2点”
14. splitDebtMaxTolerance: the max tolerance of split debt time
相对时间:
string: 1个月以后
15. splitDebtFirstPay: the first payment amount after set up split debt
string: '10,000'
*16. deltaTime: the time diff between now and contract end Date. This will be calcualted
"""
self.log = Logger(self.__class__.__name__,
level=ENV.PROFILE_LOG_LEVEL.value).logger
self.dt = LocalDateTime()
if profile is None:
self._load_default()
else:
self._load_profile(profile)
self.re_time = TimePattern()
self._loadUpLowBound()
def __init__(self,
host=None,
port=None,
debug=False,
id_increment=True,
db=None,
collection=None,
enable=True):
"""
host: mongo db host, default to chatbotdb
port: mongo db port, default to 27017
debug: False - in debug, data will be inserted into different Collections
id_increment: if the id is increased numerally. default to True
db: database default to 'chatbotdb'
collection: default to chat
"""
self.log = Logger(self.__class__.__name__,
level=ENV.DB_LOG_LEVEL.value).logger
self.debug = debug
self.enable = enable
self.id_increment = id_increment
self._load_client(host, port)
self._get_db(db)
self._get_collection(collection)
def __init__(self, start_node='s0', profile=None):
self.current_node_name = start_node
self.log = Logger(self.__class__.__name__,
level=ENV.TREE_LOG_LEVEL.value).logger
self.fc_path = []
self.all_path = []
self.profile = PF(profile)
self.conversationId = 1
self.dt = LocalDateTime()
self.cache = {
'startTime': self.dt.getLocalNow(),
'chat': [],
'nearestToleranceDate': self.profile.lowerDateTime,
'promiseToPayDate': None,
'promiseToPayAmount': 0.0
}
self.agent_response = []
class Cache:
def __init__(self,
graph_path,
msg_path,
model_dict,
max_session=1000,
debug=False):
self.max_session = 1000
self.inform_interval = 60
self.inactive_maxlength = 150
#{'uid': {'strategy': Tree(), 'time_response': <time>, 'time_inform': <>}
self.active_session = {}
self.model_dict = model_dict
self.graph_path = graph_path
self.msg_path = msg_path
self.debug = debug
self.log = Logger(self.__class__.__name__,
level=ENV.NODE_LOG_LEVEL.value).logger
self._print()
self.mongo_db = MongoClient('localhost', 27017).chatbot
def _print(self):
self.log.info('Max num of session is: {}'.format(self.max_session))
self.log.info('inform inacitve interval is {} seconds'.format(
self.inform_interval))
self.log.info('inactive max length is {} seconds'.format(
self.inactive_maxlength))
if self.debug:
self.log.info('DEBUG is enabled')
def create_session(self, uid, profile=None):
if len(self.active_session) < self.max_session:
self.active_session[uid] = {}
try:
self.active_session[uid].update({
'strategy':
TreeStage1(debug=self.debug, profile=profile)
})
except KeyError as e:
self.log.error('Key {} does not exist in profile'.format(e))
self.log.error('create session for user {} failed'.format(uid))
return False
self.active_session[uid].update({'time_response': time.time()})
self.active_session[uid].update({'time_inform': time.time()})
self.active_session[uid].update({'chatting': []})
self.log.info('New session was created: {}'.format(uid))
self.log.info('Remaining session number is: {}'.format(
self.max_session - len(self.active_session)))
return True
else:
return False
def remove_session(self, uid):
response = '您当前的会话超过 {} 秒没有响应，系统将关闭当前会话！如有需求，请开始新的对话！'.format(
self.inactive_maxlength)
try:
socketio.emit('my_response', {'data': response},
room=uid,
namespace=name_space)
except:
pass
try:
history = self.active_session[uid]['strategy'].cache.copy()
print('history', history)
print('''history['chat']''', history['chat'])
# self.mongo_db.query.insert(history) #对话结束， 已经到了删除会话的时候了，删除之前存数据，（出现end之后才存储）
except:
self.log.error('Fail to save cache for uid: ')
try:
disconnect_frontend(uid)
except:
pass
try:
del self.active_session[uid]
self.log.info(
'{} session is inactive, it has been removed!'.format(uid))
except KeyError:
pass
gc.collect()
def chat(self, uid, sentence):
if self.active_session.get(uid) is not None:
self.log.info(
'receive message from user: {} ===================='.format(
uid))
response = self.active_session[uid]['strategy'].process(
sentence, self.model_dict)
self.active_session[uid]['time_response'] = time.time()
self.active_session[uid]['time_inform'] = time.time()
self.active_session[uid]['chatting'].append(response)
self.log.info(
'processing messages for user {} has been done!----------------'
.format(uid))
else:
response = None
return response
def _bulk_deletes(self):
current = time.time()
remove_list = []
try:
for uid in self.active_session:
try:
if current - self.active_session[uid][
'time_response'] > self.inactive_maxlength:
remove_list.append(uid)
except KeyError:
pass
except RuntimeError as e:
self.log.error(e)
return False
finally:
# delete
for uid in remove_list:
self.remove_session(uid)
return True
def _bulk_inform(self):
current = time.time()
inform_list = []
try:
for uid in self.active_session:
try:
if current - self.active_session[uid][
'time_inform'] > self.inform_interval:
inform_list.append(uid)
except KeyError as e:
self.log.error(e)
pass
for each in inform_list:
self.inform_inactive(each)
except RuntimeError:
return False
return True
def purge_inactive(self):
current = time.time()
while True:
if self._bulk_deletes():
break
while True:
if self._bulk_inform():
break
def inform_inactive(self, uid):
self.active_session[uid]['time_inform'] = time.time()
response = '您有在听我说吗?请回答我刚才的问题！'
socketio.emit('my_response', {'data': response},
room=uid,
namespace=name_space)
self.log.info('{} is inactive. Just inform that user'.format(uid))
while True:
if self._bulk_inform():
break
def inform_inactive(self, uid):
self.active_session[uid]['time_inform'] = time.time()
response = '您有在听我说吗?请回答我刚才的问题！'
socketio.emit('my_response', {'data': response},
room=uid,
namespace=name_space)
self.log.info('{} is inactive. Just inform that user'.format(uid))
############### Flask ###################
logger = Logger('Websocket-Flask', level=ENV.NODE_LOG_LEVEL.value).logger
async_mode = None
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app, async_mode=async_mode)
thread = None
thread_lock = Lock()
clients = []
sentences = []
name_space = '/chat'
@app.route('/')
def index():
return render_template('index.html', async_mode=socketio.async_mode)
class Node:
def __init__(self,
node_name,
classifier=None,
msg_path=None,
canJump=False):
self.name = node_name
self._load_message(msg_path)
self.canJump = canJump
self.sentiment = 1
self.sentiment_audit = [self.sentiment]
self.model_name = classifier
self.log = Logger(self.__class__.__name__,
level=ENV.NODE_LOG_LEVEL.value).logger
def summary(self):
return {
'node_name': self.name,
'description': self.describe,
'class_name': self.__class__.__name__,
'model': self.model_name
}
def _triger_jump(self):
if self.canJump is True:
# jump trigger
if self.output_label == 1 and self.sentiment >= 3:
self.output_label = 1001
else:
return None
def process(self,
sentence,
model_dict,
lower_bound=None,
upper_bound=None):
model = model_dict[self.model_name]
clf = model.classify(sentence, lower_bound, upper_bound)
self.output_label = clf['label']
# jump trigger
self._triger_jump()
self.detail = clf
return self.output_label, self.detail
def _load_message(self, msg_path):
self.messages = pd.read_csv(msg_path, encoding='utf8')
self.messages = self.messages[self.messages['node_name'] == self.name]
self.messages.label = self.messages.label.astype('int')
self.messages.sentiment = self.messages.sentiment.astype('int')
def get_response(self, label):
"""
return response by label
"""
df = self.messages[self.messages.label == label]
max_sentiment = np.max(df.sentiment.values)
if self.sentiment > max_sentiment:
sentiment = max_sentiment
else:
sentiment = self.sentiment
df = df[df.sentiment == sentiment]
self.log.debug(
'Current sentiment is {}, node sentiment is: {}, max message sentiment is: {}'
.format(sentiment, self.sentiment, max_sentiment))
self.log.debug('Available number of message is {}'.format(len(df)))
# enable random extract
try:
df = df.sample(frac=1)
except ValueError:
response = 'current node name is{}，ouput label is{},sentiment is{}, no message has been set'.format(
self.name, label, sentiment)
self.log.error(response)
return response
try:
response = df.message.values[0]
add_sentiment = df.add_sentiment.values[0]
except IndexError:
response = 'current node name is{}，ouput label is{},sentiment is{}, no message has been set'.format(
self.name, label, sentiment)
self.log.error(response)
return response
self.sentiment += add_sentiment
self.sentiment_audit.append(self.sentiment)
return response
class PF:
def __init__(self, profile=None):
"""
profile should be None or dictionary:
fields:
1. name: lastName + firstName, eg "Li Ming"
if name is None, the constructor will try to load "lastName" and "firstName"
2. principal: the money borrowed, eg:'10,000'
3. contractStartDate, the date when money was borrowed. eg:"2018年5月2日", format"dddd年dd月dd日"
4. contractEndDate, the date before when total amount should be paid.
eg:"2018年5月2日", format"dddd年dd月dd日"
5. apr: yearly/monthly, no calculation will be involved. type: string. eg, '9%'
6. fee: late payment fee. string, eg "500"
7. lendingCompany: the money originally borrowed from
type, string, eg "平安E贷"
8. collectionCompany
type, string, eg "江苏逸能"
9. customerID
string or int "100000"
10. gender
string, "男/女"
11. collector: the agent who makes the call
string : "李明"
12. totalAmount: the total amount owed by debotor
string: “50,000”
13. informDeadline: the deadline to collect money
相对时间
string: “明天下午2点”
14. splitDebtMaxTolerance: the max tolerance of split debt time
相对时间:
string: 1个月以后
15. splitDebtFirstPay: the first payment amount after set up split debt
string: '10,000'
*16. deltaTime: the time diff between now and contract end Date. This will be calcualted
"""
self.log = Logger(self.__class__.__name__,
level=ENV.PROFILE_LOG_LEVEL.value).logger
self.dt = LocalDateTime()
if profile is None:
self._load_default()
else:
self._load_profile(profile)
self.re_time = TimePattern()
self._loadUpLowBound()
def _load_default(self):
self.log.debug(
'profile is None. The default demo profile will be loaded!')
self.name = PROFILE.lastName.value + PROFILE.firstName.value
self.principal = PROFILE.principal.value
self.contractStartDate = PROFILE.contractStartDate.value
self.contractEndDate = PROFILE.contractEndDate.value
self.apr = PROFILE.apr.value
self.interest = PROFILE.interest.value
self.fee = PROFILE.fee.value
self.lendingCompany = PROFILE.lendingCompany.value
self.collectionCompany = PROFILE.collectionCompany.value
self.customerID = PROFILE.customerID.value
self.gender = PROFILE.gender.value
self.collector = PROFILE.collector.value
self.totalAmount = PROFILE.totalAmount.value
self.informDeadline = PROFILE.informDeadline.value
self.splitDebtMaxTolerance = PROFILE.splitDebtMaxTolerance.value
self.splitDebtFirstPay = PROFILE.splitDebtFirstPay.value
self.deltaTime = (self.dt.getLocalNow() -
self.create_from_D(self.contractEndDate)).days
self._get_prefix()
self.log.info('Customer ID is {}, principal is {}, apr is {}'.format(
self.customerID, self.principal, self.apr))
def _load_profile(self, profile):
self.log.debug('Loading From Profile')
self.name = profile.get('name')
if self.name is None:
self.name = profile['lastName'] + profile['firstName']
self.principal = profile['principal']
self.contractStartDate = profile['contractStartDate']
self.contractEndDate = profile['contractEndDate']
self.apr = profile['apr']
self.interest = profile['interest']
self.fee = profile['fee']
self.lendingCompany = profile['lendingCompany']
self.collectionCompany = profile['collectionCompany']
self.customerID = profile.get('customerID')
self.gender = profile['gender']
self.collector = PROFILE.collector.value
self.totalAmount = profile['totalAmount']
self.informDeadline = profile['informDeadline']
self.splitDebtMaxTolerance = profile['splitDebtMaxTolerance']
self.splitDebtFirstPay = profile['splitDebtFirstPay']
self.deltaTime = (self.dt.getLocalNow() -
self.create_from_D(self.contractEndDate)).days
self._get_prefix()
self.log.info('Customer ID is {}, principal is {}, apr is {}'.format(
self.customerID, self.principal, self.apr))
def _loadUpLowBound(self):
upper = self.re_time.process(self.splitDebtMaxTolerance)
lower = self.re_time.process(self.informDeadline)
try:
self.upper = upper[0]['gapH']
self.upperDateTime = upper[0]['time']
self.log.info('Load profile Upper bound successfully!')
except:
self.log.error('Loading Upper error! Set to default')
self._loadDefaultUpBound()
try:
self.lower = lower[0]['gapH']
self.lowerDateTime = lower[0]['time']
self.log.info('Load profile Lower bound successfully!')
except KeyError:
self.log.error('Loading lower error! Set to default')
self._loadDefaultLowBound()
def _loadDefaultUpBound(self):
upper = self.re_time.process('1个月')
self.upper = upper[0]['gapH']
self.upperDateTime = upper[0]['time']
def _loadDefaultLowBound(self):
lower = self.re_time.process('明天下午3点')
self.lower = lower[0]['gapH']
self.lowerDateTime = lower[0]['time']
def _get_prefix(self):
if self.gender == '男':
self.prefix = '先生'
elif self.gender == '女':
self.prefix = '女士'
else:
self.prefix = '先生/女士'
def create_from_D(self, date):
year = int(re.findall('\d{4}年', date)[0][:-1])
month = int(re.findall('\d{1,2}月', date)[0][:-1])
day = int(re.findall('\d{1,2}日', date)[0][:-1])
return self.dt.createLocalTime(year=year, month=month, day=day)
class DB:
def __init__(self,
host=None,
port=None,
debug=False,
id_increment=True,
db=None,
collection=None,
enable=True):
"""
host: mongo db host, default to chatbotdb
port: mongo db port, default to 27017
debug: False - in debug, data will be inserted into different Collections
id_increment: if the id is increased numerally. default to True
db: database default to 'chatbotdb'
collection: default to chat
"""
self.log = Logger(self.__class__.__name__,
level=ENV.DB_LOG_LEVEL.value).logger
self.debug = debug
self.enable = enable
self.id_increment = id_increment
self._load_client(host, port)
self._get_db(db)
self._get_collection(collection)
def _load_client(self, host=None, port=None):
if host is None:
host = 'chatbotdb'
if port is None:
port = 27017
self.client = MongoClient(host, port)
self.log.info('get mongodb client. host is:{}, port is: {}'.format(
host, port))
def _get_db(self, db=None):
if db is None:
db = 'chatbotdb'
if self.debug:
db = db + '_debug'
self.db = self.client[db]
self.log.info('mongodb database is: {}'.format(db))
def _get_collection(self, collection=None):
if collection is None:
collection = 'chat'
if self.debug:
collection = collection + '_debug'
self.collection = self.db[collection]
self.log.info('mongodb collection is: {}'.format(collection))
def drop_collection(self):
textin = input(
'You are about to drop collection: {}. Please type in the name to confirm'
.format(self.collection.name))
if textin == self.collection.name:
self.collection.drop()
self.log.info('{} was dropped!'.format(self.collection.name))
def drop_database(self):
textin = input(
'You are about to drop database: {}. Please type in the name to confirm'
.format(self.db.name))
if textin == self.db.name:
self.client.drop_database(self.db.name)
self.log.info('DB {} was dropped!'.format(self.db.name))
def insert(self, obj={}):
if not self.enable:
self.log.warning('database is not enabled!')
return None
obj.update({'lastUpdateDate': dt.datetime.utcnow()})
if self.id_increment:
obj.update({'_id': self.collection.count_documents(filter={}) + 1})
insert_id = self.collection.insert_one(obj).inserted_id
self.log.info('{} was inserted into collection: {}'.format(
insert_id, self.collection.name))