def entityData(self, td):
data = tools.getJson(td.data)
data_text = []
for key, val in data.items():
data_text.append("%s:%s" % (key, val))
row = [tools.iso_date(td.date), ', '.join(data_text)]
return row
def testValidJson(self):
volley = [{
'json': "{}",
'to_return': {}
}, {
'json': '{"v":"1"}',
'to_return': {
"v": "1"
}
}, {
'json': '{"v":"1"\r\n}',
'to_return': {
"v": "1"
}
}, {
'json': '{"v":1}',
'to_return': {
"v": 1
}
}, {
'json': '"{}"',
'to_return': {}
}, {
'json': "invalid",
'to_return': None
}, {
'json': '[{"1":"one"}]',
'to_return': [{
1: "one"
}]
}]
for v in volley:
returned = tools.getJson(v['json'])
self.assertEqual(json.dumps(returned), json.dumps(v['to_return']))
def submit(self, d):
'''
Submit today's journal (yesterday if 00:00 - 04:00)
'''
date = None
_date = self.request.get('date')
if _date:
date = tools.fromISODate(_date)
task_json = tools.getJson(self.request.get('tasks')) # JSON
params = tools.gets(self,
strings=['lat', 'lon', 'tags_from_text'],
json=['data'],
lists=['tags'])
logging.debug(params)
if params.get('data'):
if not params.get('tags'):
params['tags'] = []
jrnl = MiniJournal.Create(self.user, date)
jrnl.Update(**params)
jrnl.parse_tags()
jrnl.put()
if task_json:
# Save new tasks for tomorrow
tasks = []
for t in task_json:
if t:
task = Task.Create(self.user, t)
tasks.append(task)
ndb.put_multi(tasks)
self.success = True
self.message = "Journal submitted!"
self.set_response({'journal': jrnl.json() if jrnl else None})
def apiai_request(self, d):
'''
'''
auth_key = self.request.headers.get('Auth-Key')
res = {'source': 'Flow'}
speech = None
end_convo = False
data = {}
if auth_key == secrets.API_AI_AUTH_KEY:
body = tools.getJson(self.request.body)
logging.debug(body)
agent_type = self._get_agent_type(body)
id, action, parameters = self._get_action_and_params(body)
self._get_user(body)
if action == 'input.disconnect':
speech = "Alright, you've disconnected your Flow account"
end_convo = True
self.signout() # Clear session
else:
from services.agent import ConversationAgent
ca = ConversationAgent(type=agent_type, user=self.user)
speech, data, end_convo = ca.respond_to_action(action, parameters=parameters)
if not speech:
speech = "Uh oh, something weird happened"
res['speech'] = speech
res['displayText'] = speech
data['google'] = {
'expect_user_response': not end_convo
}
res['data'] = data
res['contextOut'] = []
self.json_out(res, debug=True)
def __init__(self,
path,
pct_size=0.03,
max_grps=5,
chiq_pv=0.05,
ifmono=True,
keepnan=True,
methods='tree'):
self.params = {}
self.params['pct_size'] = pct_size
self.params['max_grps'] = max_grps
self.params['chiq_pv'] = chiq_pv
self.params['ifmono'] = ifmono
self.params['keepnan'] = keepnan
self.params['methods'] = methods
if isinstance(path, dict):
smy = path
else:
smy = tools.getJson(path)
self.undo_list = smy['undo']
self.fill_list = smy['fill']
self.cap_list = smy['cap'] #
self.var2char_list = smy['var2char'] #should be a dictionary
self.onehot_list = smy['onehot']
self.woe_list = smy['woeCal']
def json(self):
data = {
'id': self.key.id(),
'level':self.level,
'level_name':self.print_level(),
'name': self.name,
'email':self.email,
'phone': self.phone,
'location_text': self.location_text,
'ts_created': tools.unixtime(self.dt_created),
'services_enabled': self.services_enabled,
'service_settings': tools.getJson(self.service_settings)
}
credentials = tools.getJson(self.credentials)
if credentials:
data['scopes'] = credentials.get('scopes')
return data
def handle_error(self, response):
logging.warning(response.content)
data = tools.getJson(response.content)
error = data.get('error', {})
code = error.get('code')
subcode = error.get('error_subcode')
if code == 190 and subcode == 460:
# Error validating access token: The session has been invalidated because the user
# changed their password or Facebook has changed the session for security reasons.
pass
def AllWoeCollects(self, woe_vrs_info):
"""
collect calculated woe infos
"""
if type(woe_vrs_info) is dict:
self.woeDetail = woe_vrs_info
else:
self.woeDetail = tools.getJson(
self.path + '/feature_process_methods/IVstat/woeDetail_' +
woe_vrs_info + '.json')
def entityData(self, td):
data = tools.getJson(td.data)
data_text = []
for key, val in data.items():
data_text.append("%s:%s" % (key, val))
row = [
tools.iso_date(td.date),
', '.join(data_text)
]
return row
def generate(self, d):
type = self.request.get_range('type', default=REPORT.SENSOR_DATA_REPORT)
ftype = self.request.get_range('ftype', default=REPORT.CSV)
target = self.request.get('target')
specs_json = self.request.get('specs_json')
specs = tools.getJson(specs_json)
report = Report.Create(d['enterprise'], type=type, specs=specs, ftype=ftype)
report.put()
tools.safe_add_task(backgroundReportRun, str(report.key()), target=target, _queue="worker-queue")
self.json_out(success=True, message="%s generating..." % report.title)
def __init__(self, request, type=AGENT_FBOOK_MESSENGER, user=None):
super(FacebookAgent, self).__init__(type=type, user=user)
self.message_data = {}
self.reply = None
self.md = {} # To populate with entry.messaging[0]
self.request_type = None
self.body = tools.getJson(request.body)
if not user:
self._get_fbook_user()
self._get_request_type()
logging.debug("Authenticated user: %s. Type: %s" % (self.user, self.request_type))
logging.debug(self.body)
self._process_request()
def testValidJson(self):
volley = [
{'json': "{}", 'to_return': {}},
{'json': '{"v":"1"}', 'to_return': {"v": "1"}},
{'json': '{"v":"1"\r\n}', 'to_return': {"v": "1"}},
{'json': '{"v":1}', 'to_return': {"v": 1}},
{'json': '"{}"', 'to_return': {}},
{'json': "invalid", 'to_return': None},
{'json': '[{"1":"one"}]', 'to_return': [{1: "one"}]}
]
for v in volley:
returned = tools.getJson(v['json'])
self.assertEqual(json.dumps(returned), json.dumps(v['to_return']))
def update(self, d):
'''
Update a single TrackingDay() object with properties
defined via JSON key(str) -> value(str)
'''
date = None
_date = self.request.get('date')
if _date:
date = tools.fromISODate(_date)
data_json = tools.getJson(self.request.get('data')) # JSON
td = TrackingDay.Create(self.user, date) # Get or create
if data_json:
td.set_properties(data_json)
td.put()
self.success = True
self.set_response({'tracking_day': td.json() if td else None})
def generate(self, d):
from constants import REPORT
from handlers import APIError
from tasks import backgroundReportRun
type = self.request.get_range('type')
if not type:
raise APIError("No type in report request")
ftype = self.request.get_range('ftype', default=REPORT.CSV)
specs_json = self.request.get('specs_json')
specs = tools.getJson(specs_json)
report = Report.Create(self.user, type=type, specs=specs, ftype=ftype)
report.put()
tools.safe_add_task(backgroundReportRun, report.key.urlsafe(), _queue="report-queue")
self.set_response(success=True, message="%s generating..." % report.title, data={
'report': report.json() if report else None
})
def featureSelection_AvgScore(self, top=None, ftr_c=0.65):
"""
根据不断随机的抽取特征后的各模型表现进行特征评估
"""
model_p = pd.read_table(self.path + '/feat_imps/all_auc.json',
sep=' ',
names=['files', 'train_auc', 'test_auc'])
model_p = model_p[model_p['train_auc'] >= ftr_c]
vald_records = list(model_p['files'])
for f in range(len(vald_records)):
tgt = tools.getJson(self.path + '/feat_imps/' + vald_records[f] +
'.json')
rcd = pd.DataFrame(tgt)
rcd.columns = [vald_records[f]]
rcd.sort_values(vald_records[f], ascending=False, inplace=True)
if top is not None:
rcd = rcd.iloc[:top]
if f == 0:
rlt = rcd
else:
rlt = pd.merge(left=rlt,
right=rcd,
left_index=True,
right_index=True,
how='outer')
if len(vald_records) == 0:
raise ValueError('Not Enough Model')
#return rlt
fnl = pd.DataFrame(rlt.mean(axis=1, skipna=True), columns=['avg'])
fnl['cnt'] = rlt.count(axis=1, numeric_only=True)
fnl['score'] = fnl['avg'] * fnl['cnt'].apply(lambda x: np.log(2 + x))
return fnl
def _journal(self, message=""):
DONE_MESSAGES = ["done", "that's all", "exit", "finished", "no"]
MODES = ['questions', 'tasks', 'end']
settings = tools.getJson(self.user.settings, {})
questions = settings.get('journals', {}).get('questions', [])
end_convo = False
if questions:
jrnl = MiniJournal.Get(self.user)
if jrnl:
return (JOURNAL.ALREADY_SUBMITTED_REPLY, True)
else:
if not self.cs:
self.cs = self._create_conversation_state()
self.cs.set_state('mode', 'questions')
mode = self.cs.state.get('mode')
mode_finished = False
save_response = True
last_question = None
# Receive user message
if mode == 'tasks':
is_done = message.lower().strip() in DONE_MESSAGES
mode_finished = is_done
save_response = not is_done
elif mode == 'questions':
last_q_index = self.cs.state.get('last_q_index', -1)
last_question = last_q_index == len(questions) - 1
mode_finished = last_question
save_response = True
if save_response:
successful_add = self.cs.add_message_from_user(message)
if not successful_add:
reply = self.cs.invalid_reply(
) if mode == 'questions' else JOURNAL.INVALID_TASK
return (reply, False)
mode_index = MODES.index(mode)
if mode_finished:
mode = MODES[mode_index + 1]
self.cs.set_state('mode', mode)
reply = None
# Generate next reply
if mode == 'questions':
next_q_index = last_q_index + 1
q = questions[next_q_index]
reply = q.get('text')
name = q.get('name')
response_type = q.get('response_type')
pattern = JOURNAL.PATTERNS.get(response_type)
store_number = response_type in JOURNAL.NUMERIC_RESPONSES
self.cs.expect_reply(
pattern, name,
store_number=store_number) # Store as name
self.cs.set_state('last_q_index', next_q_index)
elif mode == 'tasks':
# Ask to add tasks
tasks = self.cs.response_data.get('tasks', [])
additional = len(tasks) > 0
reply = JOURNAL.TOP_TASK_PROMPT_ADDTL if additional else JOURNAL.TOP_TASK_PROMPT
self.cs.expect_reply(JOURNAL.PTN_TEXT_RESPONSE,
'tasks',
store_array=True) # Store as name
elif mode == 'end':
# Finish and submit
task_names = []
if 'tasks' in self.cs.response_data:
task_names = self.cs.response_data.pop('tasks')
jrnl = MiniJournal.Create(self.user)
jrnl.Update(data=self.cs.response_data)
jrnl.parse_tags()
jrnl.put()
tasks = []
if task_names:
for tn in task_names:
task = Task.Create(self.user, tn)
tasks.append(task)
ndb.put_multi(tasks)
reply = "Report submitted!"
end_convo = True
if reply:
self.cs.set_message_to_user(reply)
if end_convo:
self._expire_conversation()
else:
self._set_conversation_state()
return (reply, end_convo)
else:
return ("Please visit flowdash.co to set up journal questions",
True)
Created on Mon Jan 13 21:11:19 2020
@author: zhuchang
"""
import pandas as pd
import numpy as np
import os
import json
from tqdm import tqdm
import tools
from WoeMethods import AllWoeFuncs, WoeFuncs
import FeatureStatTools
from FeatureProcess import putNaMethods
path = 'gt_big'
raw_data_file_name = 'raw_data.csv'
na_smy = tools.getJson(path + '/' + 'na_value_info.json')
#na_smy = {
# '':''}
raw = pd.read_csv(path + '/' + raw_data_file_name, header=0)
for i in na_smy.keys():
spurs = putNaMethods(na_list=[na_smy[i]])
tmp = spurs.fit(raw[[i]]).transform(raw[[i]])
raw = raw.assign(**{i: tmp[i]})
raw.to_csv(path + '/modify_data.csv', index=False)
def get_svc_settings(self, svc_key):
svc_settings = tools.getJson(self.service_settings)
return svc_settings.get(svc_key, {})
size_c=1000)
if smy_creation:
raw_data = pd.read_csv(path + '/' + raw_data_file_name, header=0)
to_drop_list = ['num']
smy = {
'label': 'label',
'dayno': 'back_time',
'int_col': [],
'float_col': [],
'str_col': [],
'toDrop': []
}
total_data = len(raw_data)
try:
js = tools.getJson(path + '/' + 'type_info.json')
except:
js = tools.getJson(path + '/' + 'type_info_sample.json')
print('generating summary')
for k, v in tqdm(js.items()):
if k == smy['label'] or k == smy['dayno']:
continue
elif k in to_drop_list:
smy['toDrop'].append({k: 'no feature'})
elif js[k]['dist'] <= 1:
smy['toDrop'].append({k: 'unique_value'})
elif js[k]['type'] == 'str' and js[k]['dist'] > 30:
smy['toDrop'].append({k: 'too much chars'})
elif raw_data[k].isnull().sum() / total_data > 0.98:
smy['toDrop'].append({k: 'too much missing'})
elif js[k]['type'] == 'str':
size_c=1000)
if smy_creation:
raw_data = pd.read_csv(path + '/' + raw_data_file_name, header=0)
to_drop_list = ['num']
smy = {
'label': 'label',
'dayno': 'back_time',
'int_col': [],
'float_col': [],
'str_col': [],
'toDrop': []
}
total_data = len(raw_data)
try:
js = tools.getJson(path + '/' + 'type_info.json')
except:
js = tools.getJson(path + '/' + 'type_info_sample.json')
print('generating summary')
for k, v in tqdm(js.items()):
if k == smy['label'] or k == smy['dayno']:
continue
elif k in to_drop_list:
smy['toDrop'].append({k: 'no feature'})
elif js[k]['dist'] <= 1:
smy['toDrop'].append({k: 'unique_value'})
elif js[k]['type'] == 'str' and js[k]['dist'] > 30:
smy['toDrop'].append({k: 'too much chars'})
elif raw_data[k].isnull().sum() / total_data > 0.98:
smy['toDrop'].append({k: 'too much missing'})
elif js[k]['type'] == 'str':
def _maybe_get_journal_questions(self):
if self.journal_questions is None:
qs = tools.getJson(self.user.settings,
{}).get('journals', {}).get('questions', [])
self.journal_questions = qs
def _maybe_get_journal_questions(self):
if self.journal_questions is None:
qs = tools.getJson(self.user.settings, {}).get('journals', {}).get('questions', [])
self.journal_questions = qs
#设置最基本的路径变量
#通过训练样本的WOE进行特征选择并最终建逻辑回归~
if_gnrt_smy = True
ifselect = True
ifandriod = True
keep_emb = True
path = 'gt_big'
version = 'level1_trainWoeCode'
raw_data_file_name = 'raw_data.csv'
#path = '../function_test/raw_data'
rnd_seed = 21
preSelect = True
fittingpart = True
oottest = True
type_check = tools.getJson(path + '/type_info.json')
#raw_data = pd.read_csv(path+'/'+raw_data_file_name, sep = ',', header = 0, dtype = {i:type_check[i]['type'] for i in type_check.keys() if type_check[i]['type'] == 'str'})
raw_data = pd.read_csv(path + '/' + raw_data_file_name, sep=',', header=0)
"""
判断是否是andriod数据
"""
if ifandriod:
raw_data = raw_data[raw_data['ft_dev_phone_brand'] != 'Apple']
version = version + '_Andr'
else:
version = version + '_Appl'
"""
读取特征相关的统计
"""
#summary.json is required, indicating modelers primary knowledge abouut features
print("Reading related information...")
from FeatureProcess import AllFtrProcess
from FeatureSelection import ModelBasedMethods
#设置最基本的路径变量
level1 = False
level2 = False
level3 = False
level4 = True
level5 = False
path = '.'
woe_vrs = 'vrs1'
raw_data_file_name = 'data.csv'
path = '../function_test/raw_data'
rnd_seed = 21
type_check = tools.getJson(path + '/type_info.json')
raw_data = pd.read_csv(path + '/' + raw_data_file_name,
sep=r'[\t,|]',
header=0,
dtype={
i: type_check[i]['type']
for i in type_check.keys()
if type_check[i]['type'] == 'str'
})
"""
提取之前计算的IV值，为后续评估做准备
"""
ivs = tools.getJson(path + '/feature_stat/ivStat.json')
ivs = pd.DataFrame(ivs).T
ivs['avg'] = ivs.mean(axis=1)
"""
