def main_preprocessing_mmc(conf, conf_model, dataset="train"):
context_size = conf_model["context_size"]
v = conf_model["verbose"]
path_model_df_prep = Path(conf_model["path"] + "df_%s_preprocessed.csv" % dataset)
if conf_model["preprocessing_%s" % dataset][0]:
path_dataprep = Path(conf["paths"]["dataprep"] + "df_%s.csv" % dataset)
df = u.load_file(path_dataprep)
df = preprocess_text_data(df, verbose=v)
u.record_file(df, path_model_df_prep)
else:
df = u.load_file(path_model_df_prep)
df["txt"] = df["txt"].map(eval)
# Building learning matrix
path_model_df_learn = conf_model["path"] + "df_%s_learning_cs%d.csv" % (dataset, context_size)
if conf_model["preprocessing_%s" % dataset][1]:
df = create_dataframe_one_line_per_date_with_context(df, context_size, verbose=v)
u.vprint("Recording data", v)
u.record_file(df, path_model_df_learn)
else:
u.vprint("Loading data in one line per date", v)
df = u.load_file(Path(path_model_df_learn))
df[["pos_moy", "part_moy", "nb_app"]] = df[["pos_moy", "part_moy", "nb_app"]].astype(float)
X = df[["txt_id", "date", "context_date", "pos_moy", "part_moy", "nb_app"]].values
if "target" in df.columns:
y = df["target"].astype(int).values
u.vprint("X shape : %s" % str(X.shape), v)
u.vprint("y shape : %s" % str(y.shape), v)
return X, y
else:
u.vprint("X shape : %s" % str(X.shape), v)
return X
def charge_out():
"""出账 """
global TODAY
sdate_path = os.path.join(USER_PATH, "usersdate.json")
usersdate_dict = utils.load_file(sdate_path)
pending_cardnum = usersdate_dict[str(TODAY.day)]
# last_month = TODAY.replace(months=-1)
# str_currentmonth = TODAY.strftime("%Y%m")
# str_lastmonth = last_month.strftime("%Y%m")
if not pending_cardnum:
return "今天没有要出账的账户"
else:
for cardnum in pending_cardnum:
num_path = os.path.join(USER_PATH, cardnum)
acc_path = os.path.join(num_path, "account.json")
bill_path = os.path.join(num_path, "bill.json")
bill_data = utils.load_file(bill_path)
acc = utils.load_file(acc_path)
# trade_data = utils.load_file(trade_path)
bill_temp = acc["credit_total"] - acc["credit_balance"]
now = arrow.now().for_json()
if bill_temp > 0:
bill_data[TODAY.format("YYYYMM")]["dept"] = bill_temp
bill_data[TODAY.format("YYYYMM")]["repayment"] = 0
bill_data[TODAY.format("YYYYMM")]["bill_time"] = now
else:
bill_data[TODAY.format("YYYYMM")]["dept"] = bill_temp
bill_data[TODAY.format("YYYYMM")]["repayment"] = 0
bill_data[TODAY.format("YYYYMM")]["bill_time"] = now
utils.dump_to_file(bill_path, bill_data)
def adjust_prediction_with_nanc_classifier(conf, conf_model, df_pred_mmc, dataset="train"):
verbose = conf_model["verbose"]
path_model_classif = conf_model["path_nan_classifier"]
nanc_classifer = u.load_file(Path(path_model_classif))
# Load dataprep reduced to where mmc has predicted na or nc
initial_columns = df_pred_mmc.columns
ids_pred_nanc = df_pred_mmc.loc[df_pred_mmc["date_consolidation_pred"].isin(["na", "nc"]), "txt_id"]
# Get raw data
path_dataprep = Path(conf["paths"]["dataprep"] + "df_%s.csv" % dataset)
df = u.load_file(path_dataprep)
df = df.loc[df["ID"].isin(ids_pred_nanc)]
# Preprocesse this data the way na_nc_classifier works
df_prep = preprocess_text_data(df, verbose)
df_prep.loc[:, "txt"] = df["txt"].map(lambda x: " ".join(x))
X = df_prep["txt"].values
# Predict
y_pred = nanc_classifer.predict(X)
# Modify inital mmc prediction
df_pred_nanc = pd.DataFrame({"txt_id" : df_prep["ID"].values, "nanc_pred" : y_pred})
df_pred_mmc = df_pred_mmc.merge(df_pred_nanc, on="txt_id", how="left")
df_pred_mmc["date_consolidation_pred"] = df_pred_mmc.apply(final_pred, axis=1)
return df_pred_mmc[initial_columns]
def calculate_interest(card_num):
num_path = os.path.join(USER_PATH, card_num)
acc_path = os.path.join(num_path, "account.json")
bill_path = os.path.join(num_path, "bill.json")
trade_path = os.path.join(num_path, "trade.json")
print(trade_path)
bill_data = utils.load_file(bill_path)
trade_data = utils.load_file(trade_path)
acc = utils.load_file(acc_path)
statement_date = acc["STATEMENT_DATE"]
now_time = arrow.now()
bill_date = now_time.replace(months=-1, day=statement_date)
bill_key = bill_date.format("YYYYMM")
if bill_data.get(bill_key):
lower_str = bill_data[bill_key]["bill_time"]
lower_time = arrow.Arrow.strptime(lower_str, "%Y-%m-%d %H:%M:%S")
else:
return "没有账单"
repay_total = 0
for k in trade_data:
trade_time = arrow.Arrow.strptime(k, "%Y-%m-%d %H:%M:%S")
if lower_time < trade_time <= now_time:
if trade_data[k]["trade_flag"] == "0":
repay = trade_data[k]["amount"]
repay_total += repay
last_dept = bill_data[bill_key]["dept"]
last_repayment = bill_data[bill_key]["repayment"]
startday = now_time.replace(months=-2, day=statement_date)
endday = now_time
days = endday - startday
# print(days.days)
if last_dept > 0:
if last_dept <= last_repayment:
return "已还清"
else:
commission = 0
interest = (last_dept - last_repayment) * \
settings.EXPIRE_DAY_RATE * days.days
# print(interest)
interest = round(interest, 2)
# print(interest)
if repay_total / last_dept < 0.1:
if bill_data[bill_key]["commission_flag"] == 0:
repay_min = last_dept * 0.1
commission = (repay_min - repay_total) * 0.05
trade_key = now_time.strftime("%Y-%m-%d %H:%M:%S")
trade_data[trade_key] = dict()
trade_data[trade_key]["opera_type"] = "1"
trade_data[trade_key]["trade_flag"] = "1"
trade_data[trade_key]["amount"] = 0
trade_data[trade_key]["interest"] = interest
trade_data[trade_key]["commission"] = commission
utils.dump_to_file(bill_path, bill_data)
utils.dump_to_file(trade_path, trade_data)
else:
return "欠款为0"
def main_na_nc_classifier(conf):
conf_model = conf["models"]["na_nc_classifier"]
v = conf_model["verbose"]
# Preprocessing
path_model_df_prep = Path(conf_model["path"] + "df_train_preprocessed.csv")
if conf_model["preprocessing"]:
path_dataprep = Path(conf["paths"]["dataprep"] + "df_train.csv")
df_train = u.load_file(path_dataprep)
df_train = preprocessing_data(df_train, verbose=v)
u.record_file(df_train, path_model_df_prep)
else:
df_train = u.load_file(path_model_df_prep)
X = df_train["txt"].values
y = df_train["date_consolidation"].values
### Split train, test
u.vprint("Splitting data in train and test", v)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
### Learning
# Get the estimator
u.vprint("Initializing estimator", v)
estimator = get_estimator(conf_model)
# Grid search
exp_dir = get_experiment_directory(conf_model)
if conf_model["search_best_params"]:
u.vprint("Performing best params search", v)
best_params = search_best_params(conf_model, estimator, X_train,
y_train)
u.record_file(best_params, exp_dir / "best_params.json")
# Set params
estimator = set_estimator_params(estimator, conf_model, exp_dir)
# Learning
path_pickle_model = exp_dir / "fitted_model.pkl"
if conf_model["learning"]:
estimator.fit(X_train, y_train)
u.record_file(estimator, path_pickle_model)
# Assessing
res1_train, res1_val = cross_validate_model(conf_model, estimator, X_train,
y_train)
u.vprint("Cross validation results : ", v)
print(res1_train)
print(res1_val)
res_test = eval_model(estimator, X_train, y_train, X_test, y_test)
u.vprint("Test results : ", v)
print(res_test)
def create_account(card_num,
username,
role="user",
credit_total=settings.CREDIT_TOTAL,
pwd=settings.DEFAULT_PWD,
statement_date=settings.STATEMENT_DATE):
acc_path = os.path.join(USER_PATH, card_num, "account.json")
name_path = os.path.join(USER_PATH, "username.json")
sdate_path = os.path.join(USER_PATH, "usersdate.json")
num_path = os.path.join(USER_PATH, card_num)
username_list = utils.load_file(name_path)
usersdate = utils.load_file(sdate_path)
today = arrow.now()
card_num = utils.to_num(card_num)
repayment_date = today.replace(days=+settings.GRACE_PERIOD).format("DD")
if not card_num:
# print("卡号必须是16位数字")
return "卡号必须是16位数字"
if username in username_list:
# print("用户名已存在")
return "用户名已存在"
mkdir_ret = utils.mkdir(num_path)
if mkdir_ret:
# print("账户已存在")
return "账户已存在"
else:
acc_info = {
"num": card_num,
"username": username,
"role": role,
"credit_total": credit_total,
"credit_balance": credit_total,
"pwd": utils.encrypt(pwd),
"state": 0,
"deposit": 0,
"statement_date": statement_date,
"repayment_date": repayment_date
}
username_list.append(username)
usersdate[str(statement_date)].append(card_num)
utils.dump_to_file(sdate_path, usersdate)
utils.dump_to_file(acc_path, acc_info)
utils.dump_to_file(name_path, username_list)
card_path = os.path.join(USER_PATH, "card.json")
if os.path.exists(card_path):
card = utils.load_file(card_path)
else:
card = list()
card.append(card_num)
utils.dump_to_file(card_path, card)
def add_to_cart(dir1, dir2, dir1_dict, dir2_dict, chogood, good_dict, buy_num):
"""
将商品添加至购物车
:param goods_file:商品数据文件
:param username:接收已登录的用户
:param input_dir1_num:用户输入的一级目录编号
:param input_dir2_num:用户输入的二级目录编号
:param input_pro_num:用户输入的商品编号
:param input_pro_cou:接收用户输入的购买数量
:return:添加成功-True，商品编号不正确-False，库存不够-"loq"
"""
global CART
goods_dict = utils.load_file(GOODS_PATH)
good = good_dict[chogood]
g_info = goods_dict[dir1_dict[dir1]][dir2_dict[dir2]][good_dict[chogood]]
if buy_num.isdigit():
buy_num = int(buy_num)
g_qty = g_info["qty"]
if buy_num <= g_qty:
g_price = g_info["price"]
if CART.get(good_dict[chogood]):
CART[good]["qty"] += buy_num
else:
CART[good] = {}
CART[good]["price"] = g_price
CART[good]["qty"] = buy_num
goods_dict[dir1_dict[dir1]][dir2_dict[dir2]][
good_dict[chogood]]["qty"] -= buy_num
utils.dump_to_file(GOODS_PATH, goods_dict)
return showcart()
else:
return "库存不足"
else:
return "数量必须为数字"
def showdir2(dir1, dir1_dict):
"""
输出商品小类
:param goods_file:接收存有商品数据的文件
:param dir_1:接收一级目录列表
:param input_dir1:接收用户输入的一级目录
:return:返回商品二级目录
"""
goods_dict = utils.load_file(GOODS_PATH)
dir2 = str()
if dir1_dict.get(dir1):
dir2_info = goods_dict[dir1_dict[dir1]]
else:
return showdir1()
dir2_dict = {}
num = 1
for key in dir2_info:
dir2 += "%s: %s " % (num, key)
dir2_dict[str(num)] = key
num += 1
chodir2 = input(dir2)
if dir2_dict.get(chodir2):
print(dir2_dict)
return showgoods(dir1, chodir2, dir1_dict, dir2_dict)
if chodir2 == "b":
return showdir1()
def deposit(card_num, amount_of_money):
"""存款"""
acc_path = os.path.join(USER_PATH, card_num, "account.json")
acc = utils.load_file(acc_path)
acc["deposit"] += amount_of_money
utils.dump_to_file(acc_path, acc)
system.recode_trade(card_num, "0", "0", amount_of_money)
def register(username, password):
"""
用户注册
:param username:用户名
:param password:密码
:return:注册成功返回username，失败返回False
"""
patt1 = "[^(\w|_)]"
patt2 = "^[^a-zA-Z]+"
m = hashlib.md5()
m.update(bytes(password, encoding='utf-8'))
m_password = m.hexdigest()
if re.search(patt1, username) or re.match(patt2, username):
print("用户名只能为英文字母开头的数字、字母、下划线的组合！")
return False
else:
acc = utils.load_file(ACC_PATH)
if acc.get(username):
print("用户名已存在！")
return False
else:
acc[username] = {}
acc[username]["username"] = username
acc[username]["pwd"] = m_password
acc[username]["balance"] = 0
acc[username]["locked_flag"] = 0
utils.dump_to_file(ACC_PATH, acc)
return username
def setEditorTextCode(self, path):
"""设置加载合约的路径信息"""
self._editor["path"] = path
if os.path.isfile(path):
self._editor["code"] = load_file(path)
else:
self._editor["code"] = ""
def billing_query(card_num, date):
"""账单查询"""
bill_path = os.path.join(USER_PATH, card_num, "bill.json")
bill = utils.load_file(bill_path)
bill_data = bill[date]
dept = bill_data["dept"]
repayment = bill_data["repayment"]
return dept, repayment
def lock(username):
ad_path = os.path.join(DB_PATH, "credit", "admin", "admin.json")
ad_info = utils.load_file(ad_path)
if ad_info.get(username):
ad_info[username]["lock_flag"] = 1
utils.dump_to_file(ad_path, ad_info)
else:
return "不存在此管理员"
def del_account(card_num):
num_path = os.path.join(USER_PATH, card_num)
acc_path = os.path.join(num_path, "account.json")
# name_path = os.path.join(USER_PATH, "username.json")
sdate_path = os.path.join(USER_PATH, "usersdate.json")
acc = utils.load_file(acc_path)
# name_list = utils.load_file(name_path)
sdate = utils.load_file(sdate_path)
balance = acc["credit_balance"]
if balance < acc["credit_total"]:
return "该用户有欠款未还清，不能注销！"
else:
acc["state"] = 2
for d in sdate:
num_list = sdate[d]
if card_num in num_list:
num_list.remove(card_num)
utils.dump_to_file(acc_path, acc)
utils.dump_to_file(sdate_path, sdate)
def lock(username):
num_path = os.path.join(DB_PATH, "credit", "users", username)
if os.path.exists(num_path):
acc_path = os.path.join(DB_PATH, "credit", "users", username,
"account.json")
else:
return "账户不存在"
acc_info = utils.load_file(acc_path)
acc_info[username]["state"] = 1
utils.dump_to_file(acc_path, acc_info)
def load_preprocessed_data(self, *args):
preprocess_dir = self.conf["paths"]["preprocessed_data"]
data = []
for filename in args:
data_ = u.load_file(Path(preprocess_dir) / filename)
data.append(data_)
if len(data) > 1:
return data
else:
return data[0]
def transfer_accounts(to_num, amount_of_money):
"""转账"""
global USER_INFO
num = USER_INFO["username"]
acc_path = os.path.join(USER_PATH, num)
to_acc_path = os.path.join(USER_PATH, to_num)
acc_path = os.path.join(acc_path, "account.json")
to_acc_path = os.path.join(acc_path, "account.json")
acc = utils.load_file(acc_path)
try:
to_acc = utils.load_file(to_acc_path)
except Exception as e:
print(e)
if acc["deposit"] + acc["credit_balance"] >= amount_of_money:
temp = to_acc["credit_balance"] + amount_of_money
commission = 0
if acc["deposit"] >= amount_of_money:
acc["deposit"] -= amount_of_money
if temp <= to_acc["credit_total"]:
to_acc["credit_balance"] += amount_of_money
else:
temp1 = temp - to_acc["credit_total"]
to_acc["credit_balance"] = to_acc["credit_total"]
to_acc["deposit"] += temp1
else:
temp2 = amount_of_money - acc["deposit"]
acc["deposit"] = 0
commission = temp2 * settings.FETCH_MONEY_RATE
acc["credit_balance"] -= temp2 + commission
if temp <= to_acc["credit_total"]:
to_acc["credit_balance"] += amount_of_money
else:
temp3 = temp - to_acc["credit_total"]
to_acc["credit_balance"] = to_acc["credit_total"]
to_acc["deposit"] += temp3
utils.dump_to_file(acc_path, acc)
utils.dump_to_file(to_acc_path, to_acc)
system.recode_trade(num, "1", "1", amount_of_money, commission)
system.recode_trade(to_num, "0", "0", amount_of_money)
else:
return "余额不足！"
def prepare_words():
"""
Function to be called during server init.
Loads the word to ID and ID to word dictionaries, in keeping with model vocabulary.
"""
global word_to_id,id_to_word
if DEBUG >=2: print("Loading vocabulary from disk")
vocab = utils.load_file(VOCAB_FILE)
for index,v in enumerate(vocab):
word_to_id[v] = index
id_to_word[index] = v
def locking(username):
"""
锁定用户函数
:param acc_file:账户文件
:param username:用户名
:return:
"""
global ACC_PATH
acc = utils.load_file(ACC_PATH)
user_info = acc[username]
user_info["locked_flag"] = 1
utils.dump_to_file(ACC_PATH, acc)
def setEditorTextCode(self, path):
"""设置加载合约的路径信息"""
try:
if os.path.isfile(path):
code = load_file(path)
else:
code = ""
except Exception as e:
self._logger.error("打开策略失败！！")
traceback.print_exc(e)
else:
self._editor["path"] = path
self._editor["code"] = code
def repayment(card_num, amount_of_money):
"""还款"""
acc_path = os.path.join(USER_PATH, card_num, "account.json")
acc = utils.load_file(acc_path)
amounts_owed = acc["credit_total"] - acc["credit_balance"]
if amount_of_money >= amounts_owed:
acc["credit_balance"] = acc["credit_total"]
acc[deposit] += (amount_of_money - amounts_owed)
utils.dump_to_file(acc_path, acc)
else:
acc["credit_balance"] += amount_of_money
utils.dump_to_file(acc_path, acc)
system.recode_trade(card_num, "0", "0", amount_of_money)
def modify_account(card_num, new_total=None, new_date=None, new_state=None):
acc_path = os.path.join(USER_PATH, card_num, "account.json")
acc = utils.load_file(acc_path)
if new_total:
temp = acc["credit_total"]
ran = new_total - temp
acc["credit_total"] = new_total
acc["credit_balance"] += ran
if new_date:
acc["STATEMENT_DATE"] = new_date
if new_state:
acc["state"] = new_state
utils.dump_to_file(acc_path)
def init_train(self, train_path, data_path="store/train_data.pkl"):
"""初始化数据"""
try:
print("开始载入初始化数据....", file=sys.stderr)
train_data = load_file(data_path)
except BaseException:
train_data = dict()
for line in process_data.read_file(filename=train_path):
arr = line.split('\t')
train_data.setdefault(int(arr[0]), set())
train_data[int(arr[0])].add(int(arr[1]))
save_file(data_path, train_data)
return train_data
def train(self, sim_matrix_path="store/buy_user_sim.pkl"):
print("开始训练模型", file=sys.stderr)
try:
print("开始载入用户协同矩阵....", file=sys.stderr)
self.user_sim_matrix = load_file(sim_matrix_path)
print("载入协同过滤矩阵完成", file=sys.stderr)
except BaseException:
print("载入用户协同过滤矩阵失败，重新计算协同过滤矩阵", file=sys.stderr)
# 计算用户协同矩阵
self.user_sim_matrix = self.user_similarity()
print("开始保存协同过滤矩阵", file=sys.stderr)
save_file(sim_matrix_path, self.user_sim_matrix)
print("保存协同过滤矩阵完成", file=sys.stderr)
def set_estimator_params(estimator, conf_model, exp_dir):
verbose = conf_model["verbose"]
path_best_params = exp_dir / "best_params.json"
if (len(conf_model["params"]) != 0) and (not conf_model["search_best_params"]):
u.vprint("Using param from conf file : \n %s" % (str(conf_model["params"])), verbose)
estimator.set_params(**conf_model["params"])
elif path_best_params.exists():
u.vprint("Loading best params", verbose)
best_params = u.load_file(exp_dir / "best_params.json")
u.vprint(str(best_params), verbose)
estimator.set_params(**best_params)
else:
# Default parameters
u.vprint("Using default params", verbose)
u.vprint(str(estimator.get_params()), verbose)
return estimator
def spend(card_num, card_pwd, amount_of_money):
"""消费"""
acc_path = os.path.join(USER_PATH, card_num, "account.json")
acc = utils.load_file(acc_path)
if acc["state"] == 0:
if card_pwd != acc["pwd"]:
return "支付密码不正确"
else:
if acc["credit_balance"] >= amount_of_money:
acc["credit_balance"] -= amount_of_money
utils.dump_to_file(acc_path, acc)
system.recode_trade(card_num, "1", "1", amount_of_money)
else:
return "额度不足"
else:
return "您的账户已冻结"
def login(username, password):
"""用户登录
:param acc_file:账户文件
:param username:用户名
:param password:密码
:return:成功-True，密码不对-False，用户不存在-None,已锁定-"locked"
"""
password = utils.encrypt(password)
acc = utils.load_file(ACC_PATH)
if not acc.get(username):
return "用户名不存在！"
else:
if username == acc[username]["username"] and\
password == acc[username]["pwd"]:
return acc[username]
else:
return "用户名或密码不正确"
def recode_trade(card_num, trade_flag, opera_type, amount, commission=0):
"""记录操作流水"""
num_path = os.path.join(USER_PATH, card_num)
trade_path = os.path.join(num_path, "trade.json")
timenow = arrow.now().strftime("%Y-%m-%d %H:%M:%S")
if os.path.exists(trade_path):
trade = utils.load_file(trade_path)
else:
trade = dict()
# print(trade)
trade[timenow] = dict()
trade[timenow]["trade_flag"] = trade_flag
trade[timenow]["opera_type"] = opera_type
trade[timenow]["amount"] = amount
trade[timenow]["commission"] = commission
print(trade_path)
utils.dump_to_file(trade_path, trade)
def eval_model_text_level(conf_model, estimator, X_train, X_test, conf=None):
adjust_with_nanc_classifier = conf_model["adjust_with_nanc_classifer"]
path_model_df_prep = Path(conf_model["path"] + "df_train_preprocessed.csv")
df = u.load_file(path_model_df_prep, usecols=["ID", "date_accident", "date_consolidation"])
df = df.rename(columns={"ID": "txt_id"})
df_y_pred_test = predict_text_level(conf_model, estimator, X_test, adjust_with_nanc_classifier, conf)
df_y_test = df[df["txt_id"].isin(df_y_pred_test["txt_id"])].drop_duplicates().copy()
acc1_test, acc2_test, mean_acc_test = compute_accuracy(df_y_test, df_y_pred_test)
df_y_pred_train = predict_text_level(conf_model, estimator, X_train, adjust_with_nanc_classifier, conf)
df_y_train = df[df["txt_id"].isin(df_y_pred_train["txt_id"])].drop_duplicates().copy()
acc1_train, acc2_train, mean_acc_train = compute_accuracy(df_y_train, df_y_pred_train)
res3 = "Train : accuracy = %.2f (accident = %.2f, consolidation = %.2f)\n" \
"Test : accuracy = %.2f (accident = %.2f, consolidation = %.2f)" % \
(mean_acc_train, acc1_train, acc2_train, mean_acc_test, acc1_test, acc2_test)
return res3
def showdir1():
"""
输出商品一级目录
:param goods_file:接收存有商品数据的文件
:return: 返回商品的一级目录
"""
goods_dict = utils.load_file(GOODS_PATH)
dir1 = str()
num = 1
dir1_dict = {}
for key in goods_dict:
dir1 += "%s: %s " % (num, key)
dir1_dict[str(num)] = key
num += 1
chodir1 = input(dir1)
if chodir1 == "b":
return sp.main()
return showdir2(chodir1, dir1_dict)
