def cli(ctx):
# context type definition
ctx.ensure_object(dict)
# add client to context
config = Config.load()
if config is None:
ctx.obj['client'] = None
else:
client = new_client(config)
ctx.obj['client'] = client
def mongod():
"""
Initialize MongoDB.
"""
client = pymongo.MongoClient()
q1 = EventQueue()
q2 = EventQueue()
q = {'mkt': q1, 'bar': q2}
oApi = PyApi(Config(), q)
_set_db_index(oApi)
_get_all(oApi)
def test_stream():
"""
"""
q1 = EventQueue()
q2 = EventQueue()
q = {'mkt': q1, 'bar': q2}
api = PyApi(Config(), q)
mystrat = BuyAndHold(api)
q1.bind('ETYPE_MKT', api.on_market_impulse)
q2.bind('ETYPE_BAR', mystrat.on_bar)
q1.open()
q2.open()
api.make_stream('EUR_USD')
class SentenceSimpleAnalysis(object):
segmentor = Config().getSegmentor()
logger = logging.getLogger('django.request')
chckWords = [
u'查', u'查询', u'看', u'看看', u'多少', u'查查', u'告诉', u'知道', u'问', u'问问'
]
accnWords = [u'账户', u'账号', u'卡', u'余额', u'银行卡', u'存折']
frwdWords = [u'转', u'转钱', u'付', u'给', u'付给', u'打', u'转账给', u'转给', u'打给']
exchWords = [u'交易记录', u'交易', u'记录', u'消费记录', u'消费', u'收入记录', u'收入', \
u'转账记录', u'交易历史', u'交易历史记录', u'支出历史', \
u'支出历史记录', u'收入历史', u'收入历史记录', u'转账历史', u'转账历史记录', \
u'花哪里', u'花掉']
chckCommon = []
exchCommon = []
chckfile = "/var/www/BE/api/checkcomm.txt"
exchfile = "/var/www/BE/api/exchangecomm.txt"
chckfd = open(chckfile, 'r')
exchfd = open(exchfile, 'r')
for line in chckfd:
chckCommon.append(line)
for line in exchfd:
exchCommon.append(line)
symbols = set([u'，', u'。', u'！', u'？'])
virtualwords = [
u'吧', u'啊', u'吗', u'儿', u'请', u'帮忙', u'帮', u'一下', u'想', u'想要', u'想让你',
u'我的', u'我', u'个', u'去', u'到', u'还', u'是', u'里面', u'里', u'都', u'了'
]
chck_del_words = [u'是', u'钱']
def __init__(self):
self.chckSentences = []
self.frwdSentences = []
self.exchSentences = []
def __delCheckWords(self, sentence):
temp = sentence
for del_word in self.chck_del_words:
temp = temp.replace(del_word, '')
return temp
def delNonUsedWords(self, sentence):
for symbol in self.symbols:
sentence = sentence.replace(symbol, '')
for virtualword in self.virtualwords:
sentence = sentence.replace(virtualword, '')
return sentence
def commClass(self, sentence1):
sentence = self.delNonUsedWords(sentence1)
self.logger.debug(sentence)
for line in self.chckCommon:
line = line.replace('\n', '')
if sentence == line:
return [1]
for line in self.exchCommon:
line = line.replace('\n', '')
linelist = line.split('\t')
if linelist[0] == sentence:
result = linelist[1].split('%')
index = 0
for res in result:
if res.isdigit():
result[index] = int(res)
index += 1
return result
return False
def cutWords(self, sentence):
# can be replaced by pyltp
wordlist = self.segmentor.segment(sentence)
result = list(wordlist)
return result
def cutSentence(self, text):
result = SnowNLP(text)
sentences = result.sentences
return sentences
def isChckWords(self, wordlist):
chckCata = False
for ll in self.chckWords:
for ww in wordlist:
if ww.find(ll) != -1:
chckCata = True
break
accnCata = False
for ll2 in self.accnWords:
for ww2 in wordlist:
if ww2.find(ll2) != -1:
accnCata = True
break
if accnCata:
return True
if (chckCata & accnCata):
return True
return False
def isFrwdWords(self, wordlist):
for ll in self.frwdWords:
for ww in wordlist:
if ww.find(ll) != -1:
return True
return False
def isExchWords(self, wordlist):
for ll in self.exchWords:
for ww in wordlist:
if ww.find(ll) != -1:
return True
return False
def getEffectSentence(self, sentences):
for sentence in sentences:
self.logger.debug(sentence)
commands = self.delNonUsedWords(sentence)
wordlist = self.cutWords(commands)
if self.isExchWords(wordlist):
self.exchSentences.append(wordlist)
elif self.isChckWords(wordlist):
tempstr = "".join(wordlist)
tempstr = self.__delCheckWords(tempstr)
wordlist = self.cutWords(tempstr)
self.chckSentences.append(wordlist)
elif self.isFrwdWords(wordlist):
self.frwdSentences.append(wordlist)
else:
continue
def processSentence(self, rawsentence):
sentences = self.cutSentence(rawsentence)
self.logger.debug("sentences is " + str(sentences))
if (len(sentences) == 1):
comm_result = self.commClass(sentences[0])
if comm_result:
return comm_result
self.getEffectSentence(sentences)
# print (effsentences)
if (len(self.chckSentences) == 0 and len(self.frwdSentences) == 0
and len(self.exchSentences) == 0):
return [0]
else:
self.logger.debug("Go to further process.")
if len(self.chckSentences):
sda = SentenceDeepAnalysis(self.chckSentences)
self.logger.debug("Go to checkRemainAmount")
result = sda.checkRemainAmount(self.chckSentences)
self.chckSentences = []
return result
if len(self.frwdSentences):
sda = SentenceDeepAnalysis(self.frwdSentences)
self.logger.debug("Go to forwardMoney")
result = sda.forwardMoney(self.frwdSentences)
self.frwdSentences = []
return result
if len(self.exchSentences):
sda = SentenceDeepAnalysis(self.exchSentences)
self.logger.debug("Go to checkAccountExch")
result = sda.checkAccountExch(self.exchSentences)
self.exchSentences = []
return result
return [0]
start = datetime.strptime(start, '%Y%m%d %H:%M')
end = datetime.strptime(end, '%Y%m%d %H:%M')
else:
pass
docs = []
# find in MongoDB.
for doc in coll.find(
filter={dbIndex: {
'$lte': end,
'$gte': start
}},
projection={'_id': False}):
docs.append(doc)
if output == 'list':
return docs[::-1]
except Exception as e:
msg = '[MONGOD]: Error encountered when fetching data' + \
'from MongoDB; '+ str(e)
return -1
if __name__ == '__main__':
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.update_index_D1()
def config_new(ctx):
api_key = click.prompt('API Key', hide_input=True, type=str)
token = click.prompt('API Token', hide_input=True, type=str)
cfg = Config(api_key=api_key, token=token)
cfg.write_to_yaml()
class SentenceDeepAnalysis(object):
config = Config()
logger = logging.getLogger("django.request")
segmentor = config.getSegmentor()
parser = config.getParser()
postagger = config.getPostagger()
recognizer = config.getNameRecognizer()
chech_verbs = [
u'查', u'查询', u'看', u'看看', u'多少', u'查查', u'问问', u'问', u'有', u'剩'
]
negative_advs = [u'不', u'别', u'非', u'否', u'不该', u'勿']
question_words = [u'为什么', u'怎么', u'怎么就', u'为何', u'如何']
accn_words = [u'账户', u'账号', u'卡', u'余额', u'银行卡', u'卡里', u'多少', u'存折']
pls_words = [u'给', u'把']
frwd_words = [
u'转', u'转账', u'转钱', u'付', u'給', u'付给', u'打', u'转给', u'打给', u'花', u'收入',
u'支付'
]
exch_words = [
u'交易记录', u'交易', u'记录', u'消费记录', u'消费', u'收入记录', u'收入', u'哪', u'钱'
]
num_words = [
u'零', u'一', u'二', u'三', u'四', u'五', u'六', u'七', u'八', u'九', u'十一',
u'十二'
]
unit_words = [u'万', u'千', u'百', u'十']
time_units = [u'年', u'月', u'天', u'日', u'次', u'笔']
eng_time_units = ['y', 'm', 'd', 'd', 't', 't']
time_pos_pat = [('nd', 'nt'), ('nt', ), ('nd', ), ('nt', 'nt')]
nd_keywords = [u'前', u'上', u'最近', u'近']
exch_att_1 = [u'账户', u'账号', u'银行卡', u'存折', u'查', u'查询']
exch_att_2 = [u'转账记录', u'交易记录', u'消费记录', u'记录', u'交易', u'转账', u'历史']
rela_vob = 'VOB'
rela_iob = 'IOB'
rela_sob = 'SOB'
rela_pob = 'POB'
rela_sbv = 'SBV'
rela_att = 'ATT'
chck_att_1 = [u'账户', u'账号', u'卡里', u'银行卡', u'存折', u'卡', u'钱']
chck_att_2 = [u'余额', u'多少', u'钱']
frwd_att_1 = [
u'转', u'转账', u'转钱', u'付', u'給', u'付给', u'打', u'转给', u'打给', u'花', u'收入',
u'支付'
]
def __init__(self, effSentence):
self.sentence = effSentence
self.username = ''
def __concatSeq(self, seq):
concated_seq = None
if seq:
if (len(seq) > 1):
concated_seq = " ".join(seq)
else:
concated_seq = seq[0]
return concated_seq
def __ifSeqinSeq(self, seqstr1, seqstr2):
matchpos = seqstr1.find(seqstr2)
if matchpos >= 1:
return matchpos - 1
else:
return "None"
def __getSpeciPos(self, words, pos, po_char):
speci_words = []
index = 0
for po in pos:
if po == po_char:
speci_words.append(words[index])
index = index + 1
return speci_words
def __getTimeUnit(self, nt_word):
index = 0
for time_unit in self.time_units:
if nt_word.find(time_unit) != -1:
return self.eng_time_units[index]
index = index + 1
return "None"
def __getTimeUnit2(self, nt_word):
index = 0
for time_unit in self.time_units:
if nt_word.find(time_unit) != -1:
return self.time_units[index]
index = index + 1
return "None"
def __timeToList(self, nt_word):
to_list = []
word = nt_word
time_unit = self.__getTimeUnit(word)
time_unit2 = self.__getTimeUnit2(word)
if time_unit != "None":
to_list.insert(0, time_unit)
word = word.replace(time_unit2, '')
# print (word)
num = self.__mapWordNum(word)
to_list.insert(0, num)
return to_list
else:
return None
def __getQuaWords(self, words, pos):
m_words = []
if words and pos:
index = 0
for po in pos:
if po == 'm':
m_words.append(words[index])
index += 1
return m_words
def __getQuaUnits(self, words, pos):
q_words = []
if words and pos:
index = 0
for po in pos:
if po == 'q':
q_words.append(words[index])
index += 1
return q_words
# time_pos_pat = [('m', 'm', 'q', 'nt', 'nt'),('m', 'q', 'nt', 'nt'),('m', 'q', 'nt'),('nt', 'nt', 'nt'),('nt','nt'),('nd','nt'),('nt')]
def __getTimeInfo(self, words, pos):
nt_words = self.__getSpeciPos(words, pos, 'nt')
nd_words = self.__getSpeciPos(words, pos, 'nd')
if (len(nd_words) > 1 or len(nt_words) > 1):
return [99, "无法定位到您想要的时间点"]
if (len(nd_words) == 1) and (nd_words[0] in self.nd_keywords):
if len(nt_words) == 1:
time_list = self.__timeToList(nt_words[0])
return time_list
else:
return [99, "无法定位到您想要的时间点"]
if (len(nd_words) == 1 and len(nt_words) == 0 and nd_words[0] in self.nd_keywords) \
or (len(nt_words) == 1 and len(nd_words) == 0 and nt_words[0] in self.nd_keywords):
m_words = self.__getQuaWords(words, pos)
q_words = self.__getQuaUnits(words, pos)
if (len(m_words) == 1 and len(q_words) == 1):
m_num = self.__mapWordNum(m_words[0])
q_num = self.__getTimeUnit(q_words[0])
return [m_num, q_num]
return [99, "无法定位到您想要的时间点"]
def __getNameinSent(self, words, pos):
names = self.recognizer.recognize(words, pos)
pers_name = []
index = 0
for name in names:
if name == 'S-Ni' or name == 'S-Nh':
pers_name.append(words[index])
index = index + 1
index = 0
if len(pers_name) == 0:
for po in pos:
if po == 'nh':
pers_name.append(words[index])
index = index + 1
return pers_name
def __getVerbinSent(self, words, pos):
index = 0
verbs = []
for po in pos:
if po == 'v':
verbs.append(words[index])
index = index + 1
return verbs
def __getAdvinSent(self, words, pos):
index = 0
advs = []
for po in pos:
if po == 'd':
advs.append(words[index])
index = index + 1
return advs
def __getPlsinSent(self, words, pos):
index = 0
advs = []
for po in pos:
if po == 'p':
advs.append(words[index])
index = index + 1
return advs
def __getRawSentence(self, words):
sentence = ''
for word in words:
sentence = sentence + str(word)
return sentence
def __getWordRelation(self, words, pos):
arcs = self.parser.parse(words, pos)
return arcs
def __getAllRelation(self, arcs):
all_relation = []
if arcs:
for arc in arcs:
all_relation.append(arc.relation)
return all_relation
def __getRelationID(self, arcs, relation):
relay_ids = []
index = 0
for arc in arcs:
if arc.relation == relation:
relay_ids.append(index)
index = index + 1
return relay_ids
def __getRelationHead(self, words, arcs, relay_id):
return words[arcs[relay_id].head - 1]
def __getPostags(self, words):
return self.postagger.postag(words)
def __checkFrwdAmount(self, words, pos):
frwd_amount = []
m_word = ''
index = 0
for po in pos:
if po == 'm':
m_word = words[index]
if m_word.find(u'元') or m_word.find(u'块'):
m_word = m_word.replace('元', '')
m_word = m_word.replace('块', '')
frwd_amount.append(m_word)
else:
frwd_amount.append(m_word)
index = index + 1
# print (m_word)
return frwd_amount
def __mapWordNum(self, amount_word):
index = 0
if (amount_word.isdigit()):
return amount_word
if (amount_word == u'两'):
return 2
if not (amount_word):
return 1
for num_word in self.num_words:
if num_word == amount_word:
return index
index = index + 1
return None
def __mapUnitNum(self, word):
if word == u'万':
return 10000
if word == u'千':
return 1000
if word == u'百':
return 100
if word == u'十':
return 10
if word == u'零':
return 0
def __convertToNumber(self, amount_word):
if (amount_word.isdigit()):
return amount_word
word = amount_word[::-1]
word = list(word)
amount = 0
if word[0] in self.num_words:
amount += self.__mapWordNum(word[0])
word.pop(0)
while (len(word)):
unit_word = word[0]
num_word = word[1]
unit_amount = self.__mapUnitNum(unit_word)
num_amount = self.__mapWordNum(num_word)
total_amount = unit_amount * num_amount
amount += total_amount
if (unit_word == u'零'):
word.pop(0)
else:
word.pop(0)
word.pop(0)
return total_amount
def checkRemainAmount(self, sentences):
self.logger.debug(sentences)
for sentence in sentences:
pos = self.__getPostags(sentence)
person = self.__getNameinSent(sentence, pos)
rawsentence = self.__getRawSentence(sentence)
arcs = self.__getWordRelation(sentence, pos)
arc_relations = self.__getAllRelation(arcs)
if (len(person) > 1):
return [99, "你不可能查两个人的余额"]
if (len(person) != 0 and person[0] != self.username):
return [99, "你不可能查别人的余额"]
verbs = self.__getVerbinSent(sentence, pos)
self.logger.debug(verbs)
advs = self.__getAdvinSent(sentence, pos)
# print (advs)
if self.rela_vob in arc_relations and verbs:
for verb in verbs:
if verb in self.chech_verbs:
vob_relation_id = self.__getRelationID(
arcs, self.rela_vob)
for id in vob_relation_id:
vob_relation_head = self.__getRelationHead(
sentence, arcs, id)
for adv in advs:
adv_verb = str(adv) + str(verb)
adv_verb2 = str(verb) + str(adv)
if rawsentence.find(
adv_verb) == -1 and rawsentence.find(
adv_verb2) == -1:
continue
else:
if adv in self.negative_advs:
return [99, "您不想查询余额"]
else:
continue
if sentence[
id] in self.accn_words and vob_relation_head == verb:
return [1]
# index += 1
if self.rela_att in arc_relations:
att_relation_id = self.__getRelationID(arcs, self.rela_att)
for id in att_relation_id:
att_relation_head = self.__getRelationHead(
sentence, arcs, id)
if (sentence[id] in self.chck_att_1 and att_relation_head in self.chck_att_2) or \
(sentence[id] in self.chck_att_2 and att_relation_head in self.chck_att_2):
return [1]
if self.rela_sbv in arc_relations:
sbv_relation_id = self.__getRelationID(arcs, self.rela_sbv)
for id in sbv_relation_id:
sbv_relation_head = self.__getRelationHead(
sentence, arcs, id)
if (sentence[id] in self.chck_att_1 and sbv_relation_head in self.chck_att_2) or \
(sentence[id] in self.chck_att_2 and sbv_relation_head in self.chck_att_2):
return [1]
return [99, "您给的指令有误"]
def forwardMoney(self, sentences):
self.logger.debug(sentences)
for sentence in sentences:
pos = self.__getPostags(sentence)
person = self.__getNameinSent(sentence, pos)
rawsentence = self.__getRawSentence(sentence)
arcs = self.__getWordRelation(sentence, pos)
arc_relations = self.__getAllRelation(arcs)
if len(person) == 0:
return [99, "不知道您要转账给谁"]
else:
verbs = self.__getVerbinSent(sentence, pos)
pls = self.__getPlsinSent(sentence, pos)
advs = self.__getAdvinSent(sentence, pos)
for pl in pls:
verbs.insert(0, pl)
for verb in verbs:
if verb in self.frwd_words or verb == u'给':
self.logger.debug("verb " + verb + " is in frwd_words")
for adv in advs:
adv_verb = str(adv) + str(verb)
adv_verb2 = str(verb) + str(adv)
if rawsentence.find(
adv_verb) == -1 or rawsentence.find(
adv_verb2) == -1:
continue
else:
if adv in self.negative_advs:
return [99, "您不转账"]
else:
continue
vob_relation_id = self.__getRelationID(arcs, self.rela_vob)
if vob_relation_id:
# print ("VOB on " + str(vob_relation_id))
# print ("OBJ is " + str(sentence[vob_relation_id[0]]))
for id in vob_relation_id:
vob_relation_head = self.__getRelationHead(
sentence, arcs, id)
if (sentence[id].find(u'元') != -1
or sentence[id].find(u'块') != -1
or sentence[id].isdigit()
) and vob_relation_head == verb:
frwd_amount = self.__checkFrwdAmount(
sentence, pos)
if len(frwd_amount) != 1:
return [99, "缺失转账金额或没有转账金额"]
amount = self.__convertToNumber(frwd_amount[0])
return [3, person[0], amount]
pob_relation_id = self.__getRelationID(arcs, self.rela_pob)
if pob_relation_id:
pob_relation_head = self.__getRelationHead(
sentence, arcs, pob_relation_id[0])
if (sentence[pob_relation_id[0]].find(u'元') != -1
or sentence[pob_relation_id[0]].find(u'块') != -1
or sentence[pob_relation_id[0]].isdigit()
) and pob_relation_head == pls[0]:
frwd_amount = self.__checkFrwdAmount(sentence, pos)
if len(frwd_amount) != 1:
return [99, "缺失转账金额或没有转账金额"]
amount = self.__convertToNumber(frwd_amount[0])
return [3, person[0], amount]
if self.rela_att in arc_relations:
att_relation_id = self.__getRelationID(arcs, self.rela_att)
for id in att_relation_id:
att_relation_head = self.__getRelationHead(
sentence, arcs, id)
if (sentence[id] in self.frwd_att_1 and att_relation_head == person[0]) or \
(sentence[id] == person[0] and att_relation_head in self.frwd_att_1):
frwd_amount = self.__checkFrwdAmount(sentence, pos)
# print (frwd_amount)
if len(frwd_amount) != 1:
return [99, "缺失转账金额或没有转账金额"]
amount = self.__convertToNumber(frwd_amount[0])
return [3, person[0], amount]
return [99, "您给的指令有误"]
def checkAccountExch(self, sentences):
self.logger.debug(sentences)
for sentence in sentences:
pos = self.__getPostags(sentence)
person = self.__getNameinSent(sentence, pos)
rawsentence = self.__getRawSentence(sentence)
arcs = self.__getWordRelation(sentence, pos)
arc_relations = self.__getAllRelation(arcs)
if (len(person) > 1):
return [99, "你不可能查两个人的交易记录"]
if (len(person) != 0 and person[0] != self.username):
return [99, "你不可能查别人的交易记录"]
verbs = self.__getVerbinSent(sentence, pos)
advs = self.__getAdvinSent(sentence, pos)
if self.rela_vob in arc_relations and verbs:
for verb in verbs:
if verb in self.chech_verbs:
self.logger.debug("verb " + verb + " is in exch_words")
for adv in advs:
adv_verb = str(adv) + str(verb)
adv_verb2 = str(verb) + str(adv)
if rawsentence.find(
adv_verb) == -1 and rawsentence.find(
adv_verb2) == -1:
continue
else:
if adv in self.negative_advs:
return [99, "您不想查询交易记录"]
vob_relation_id = self.__getRelationID(arcs, self.rela_vob)
for id in vob_relation_id:
vob_relation_head = self.__getRelationHead(
sentence, arcs, id)
if vob_relation_id:
if sentence[
id] in self.exch_words and vob_relation_head == verb:
for pat in self.time_pos_pat:
seq_pat = self.__concatSeq(pat)
# print (seq_pat)
sentence_pat = self.__concatSeq(pos)
# print (sentence_pat)
if self.__ifSeqinSeq(
sentence_pat, seq_pat) != "None":
time_result = self.__getTimeInfo(
sentence, pos)
return [
2, time_result[0], time_result[1]
]
if self.rela_att in arc_relations:
att_relation_id = self.__getRelationID(arcs, self.rela_att)
for id in att_relation_id:
att_relation_head = self.__getRelationHead(
sentence, arcs, id)
if (sentence[id] in self.exch_att_1 and att_relation_head in self.exch_att_2) or \
(sentence[id] in self.exch_att_2 and att_relation_head in self.exch_att_2):
for pat in self.time_pos_pat:
seq_pat = self.__concatSeq(pat)
# print (seq_pat)
sentence_pat = self.__concatSeq(pos)
# print (sentence_pat)
if self.__ifSeqinSeq(sentence_pat,
seq_pat) != "None":
time_result = self.__getTimeInfo(sentence, pos)
# print (time_result)
return [2, time_result[0], time_result[1]]
if self.rela_sbv in arc_relations:
sbv_relation_id = self.__getRelationID(arcs, self.rela_sbv)
for id in sbv_relation_id:
sbv_relation_head = self.__getRelationHead(
sentence, arcs, id)
if (sentence[id] in self.exch_att_1 and sbv_relation_head in self.exch_att_2) or \
(sentence[id] in self.exch_att_2 and sbv_relation_head in self.exch_att_2):
for pat in self.time_pos_pat:
seq_pat = self.__concatSeq(pat)
# print (seq_pat)
sentence_pat = self.__concatSeq(pos)
# print (sentence_pat)
if self.__ifSeqinSeq(sentence_pat,
seq_pat) != "None":
time_result = self.__getTimeInfo(sentence, pos)
# print (time_result)
return [2, time_result[0], time_result[1]]
return [99, "指令不对，无法完成交易查询操作"]
return [99, "指令不明确，无法完成交易查询操作"]