def main_preprocessing_mmc(conf, conf_model, dataset="train"): context_size = conf_model["context_size"] v = conf_model["verbose"] path_model_df_prep = Path(conf_model["path"] + "df_%s_preprocessed.csv" % dataset) if conf_model["preprocessing_%s" % dataset][0]: path_dataprep = Path(conf["paths"]["dataprep"] + "df_%s.csv" % dataset) df = u.load_file(path_dataprep) df = preprocess_text_data(df, verbose=v) u.record_file(df, path_model_df_prep) else: df = u.load_file(path_model_df_prep) df["txt"] = df["txt"].map(eval) # Building learning matrix path_model_df_learn = conf_model["path"] + "df_%s_learning_cs%d.csv" % (dataset, context_size) if conf_model["preprocessing_%s" % dataset][1]: df = create_dataframe_one_line_per_date_with_context(df, context_size, verbose=v) u.vprint("Recording data", v) u.record_file(df, path_model_df_learn) else: u.vprint("Loading data in one line per date", v) df = u.load_file(Path(path_model_df_learn)) df[["pos_moy", "part_moy", "nb_app"]] = df[["pos_moy", "part_moy", "nb_app"]].astype(float) X = df[["txt_id", "date", "context_date", "pos_moy", "part_moy", "nb_app"]].values if "target" in df.columns: y = df["target"].astype(int).values u.vprint("X shape : %s" % str(X.shape), v) u.vprint("y shape : %s" % str(y.shape), v) return X, y else: u.vprint("X shape : %s" % str(X.shape), v) return X
def charge_out(): """出账 """ global TODAY sdate_path = os.path.join(USER_PATH, "usersdate.json") usersdate_dict = utils.load_file(sdate_path) pending_cardnum = usersdate_dict[str(TODAY.day)] # last_month = TODAY.replace(months=-1) # str_currentmonth = TODAY.strftime("%Y%m") # str_lastmonth = last_month.strftime("%Y%m") if not pending_cardnum: return "今天没有要出账的账户" else: for cardnum in pending_cardnum: num_path = os.path.join(USER_PATH, cardnum) acc_path = os.path.join(num_path, "account.json") bill_path = os.path.join(num_path, "bill.json") bill_data = utils.load_file(bill_path) acc = utils.load_file(acc_path) # trade_data = utils.load_file(trade_path) bill_temp = acc["credit_total"] - acc["credit_balance"] now = arrow.now().for_json() if bill_temp > 0: bill_data[TODAY.format("YYYYMM")]["dept"] = bill_temp bill_data[TODAY.format("YYYYMM")]["repayment"] = 0 bill_data[TODAY.format("YYYYMM")]["bill_time"] = now else: bill_data[TODAY.format("YYYYMM")]["dept"] = bill_temp bill_data[TODAY.format("YYYYMM")]["repayment"] = 0 bill_data[TODAY.format("YYYYMM")]["bill_time"] = now utils.dump_to_file(bill_path, bill_data)
def adjust_prediction_with_nanc_classifier(conf, conf_model, df_pred_mmc, dataset="train"): verbose = conf_model["verbose"] path_model_classif = conf_model["path_nan_classifier"] nanc_classifer = u.load_file(Path(path_model_classif)) # Load dataprep reduced to where mmc has predicted na or nc initial_columns = df_pred_mmc.columns ids_pred_nanc = df_pred_mmc.loc[df_pred_mmc["date_consolidation_pred"].isin(["na", "nc"]), "txt_id"] # Get raw data path_dataprep = Path(conf["paths"]["dataprep"] + "df_%s.csv" % dataset) df = u.load_file(path_dataprep) df = df.loc[df["ID"].isin(ids_pred_nanc)] # Preprocesse this data the way na_nc_classifier works df_prep = preprocess_text_data(df, verbose) df_prep.loc[:, "txt"] = df["txt"].map(lambda x: " ".join(x)) X = df_prep["txt"].values # Predict y_pred = nanc_classifer.predict(X) # Modify inital mmc prediction df_pred_nanc = pd.DataFrame({"txt_id" : df_prep["ID"].values, "nanc_pred" : y_pred}) df_pred_mmc = df_pred_mmc.merge(df_pred_nanc, on="txt_id", how="left") df_pred_mmc["date_consolidation_pred"] = df_pred_mmc.apply(final_pred, axis=1) return df_pred_mmc[initial_columns]
def calculate_interest(card_num): num_path = os.path.join(USER_PATH, card_num) acc_path = os.path.join(num_path, "account.json") bill_path = os.path.join(num_path, "bill.json") trade_path = os.path.join(num_path, "trade.json") print(trade_path) bill_data = utils.load_file(bill_path) trade_data = utils.load_file(trade_path) acc = utils.load_file(acc_path) statement_date = acc["STATEMENT_DATE"] now_time = arrow.now() bill_date = now_time.replace(months=-1, day=statement_date) bill_key = bill_date.format("YYYYMM") if bill_data.get(bill_key): lower_str = bill_data[bill_key]["bill_time"] lower_time = arrow.Arrow.strptime(lower_str, "%Y-%m-%d %H:%M:%S") else: return "没有账单" repay_total = 0 for k in trade_data: trade_time = arrow.Arrow.strptime(k, "%Y-%m-%d %H:%M:%S") if lower_time < trade_time <= now_time: if trade_data[k]["trade_flag"] == "0": repay = trade_data[k]["amount"] repay_total += repay last_dept = bill_data[bill_key]["dept"] last_repayment = bill_data[bill_key]["repayment"] startday = now_time.replace(months=-2, day=statement_date) endday = now_time days = endday - startday # print(days.days) if last_dept > 0: if last_dept <= last_repayment: return "已还清" else: commission = 0 interest = (last_dept - last_repayment) * \ settings.EXPIRE_DAY_RATE * days.days # print(interest) interest = round(interest, 2) # print(interest) if repay_total / last_dept < 0.1: if bill_data[bill_key]["commission_flag"] == 0: repay_min = last_dept * 0.1 commission = (repay_min - repay_total) * 0.05 trade_key = now_time.strftime("%Y-%m-%d %H:%M:%S") trade_data[trade_key] = dict() trade_data[trade_key]["opera_type"] = "1" trade_data[trade_key]["trade_flag"] = "1" trade_data[trade_key]["amount"] = 0 trade_data[trade_key]["interest"] = interest trade_data[trade_key]["commission"] = commission utils.dump_to_file(bill_path, bill_data) utils.dump_to_file(trade_path, trade_data) else: return "欠款为0"
def main_na_nc_classifier(conf): conf_model = conf["models"]["na_nc_classifier"] v = conf_model["verbose"] # Preprocessing path_model_df_prep = Path(conf_model["path"] + "df_train_preprocessed.csv") if conf_model["preprocessing"]: path_dataprep = Path(conf["paths"]["dataprep"] + "df_train.csv") df_train = u.load_file(path_dataprep) df_train = preprocessing_data(df_train, verbose=v) u.record_file(df_train, path_model_df_prep) else: df_train = u.load_file(path_model_df_prep) X = df_train["txt"].values y = df_train["date_consolidation"].values ### Split train, test u.vprint("Splitting data in train and test", v) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) ### Learning # Get the estimator u.vprint("Initializing estimator", v) estimator = get_estimator(conf_model) # Grid search exp_dir = get_experiment_directory(conf_model) if conf_model["search_best_params"]: u.vprint("Performing best params search", v) best_params = search_best_params(conf_model, estimator, X_train, y_train) u.record_file(best_params, exp_dir / "best_params.json") # Set params estimator = set_estimator_params(estimator, conf_model, exp_dir) # Learning path_pickle_model = exp_dir / "fitted_model.pkl" if conf_model["learning"]: estimator.fit(X_train, y_train) u.record_file(estimator, path_pickle_model) # Assessing res1_train, res1_val = cross_validate_model(conf_model, estimator, X_train, y_train) u.vprint("Cross validation results : ", v) print(res1_train) print(res1_val) res_test = eval_model(estimator, X_train, y_train, X_test, y_test) u.vprint("Test results : ", v) print(res_test)
def create_account(card_num, username, role="user", credit_total=settings.CREDIT_TOTAL, pwd=settings.DEFAULT_PWD, statement_date=settings.STATEMENT_DATE): acc_path = os.path.join(USER_PATH, card_num, "account.json") name_path = os.path.join(USER_PATH, "username.json") sdate_path = os.path.join(USER_PATH, "usersdate.json") num_path = os.path.join(USER_PATH, card_num) username_list = utils.load_file(name_path) usersdate = utils.load_file(sdate_path) today = arrow.now() card_num = utils.to_num(card_num) repayment_date = today.replace(days=+settings.GRACE_PERIOD).format("DD") if not card_num: # print("卡号必须是16位数字") return "卡号必须是16位数字" if username in username_list: # print("用户名已存在") return "用户名已存在" mkdir_ret = utils.mkdir(num_path) if mkdir_ret: # print("账户已存在") return "账户已存在" else: acc_info = { "num": card_num, "username": username, "role": role, "credit_total": credit_total, "credit_balance": credit_total, "pwd": utils.encrypt(pwd), "state": 0, "deposit": 0, "statement_date": statement_date, "repayment_date": repayment_date } username_list.append(username) usersdate[str(statement_date)].append(card_num) utils.dump_to_file(sdate_path, usersdate) utils.dump_to_file(acc_path, acc_info) utils.dump_to_file(name_path, username_list) card_path = os.path.join(USER_PATH, "card.json") if os.path.exists(card_path): card = utils.load_file(card_path) else: card = list() card.append(card_num) utils.dump_to_file(card_path, card)
def add_to_cart(dir1, dir2, dir1_dict, dir2_dict, chogood, good_dict, buy_num): """ 将商品添加至购物车 :param goods_file:商品数据文件 :param username:接收已登录的用户 :param input_dir1_num:用户输入的一级目录编号 :param input_dir2_num:用户输入的二级目录编号 :param input_pro_num:用户输入的商品编号 :param input_pro_cou:接收用户输入的购买数量 :return:添加成功-True,商品编号不正确-False,库存不够-"loq" """ global CART goods_dict = utils.load_file(GOODS_PATH) good = good_dict[chogood] g_info = goods_dict[dir1_dict[dir1]][dir2_dict[dir2]][good_dict[chogood]] if buy_num.isdigit(): buy_num = int(buy_num) g_qty = g_info["qty"] if buy_num <= g_qty: g_price = g_info["price"] if CART.get(good_dict[chogood]): CART[good]["qty"] += buy_num else: CART[good] = {} CART[good]["price"] = g_price CART[good]["qty"] = buy_num goods_dict[dir1_dict[dir1]][dir2_dict[dir2]][ good_dict[chogood]]["qty"] -= buy_num utils.dump_to_file(GOODS_PATH, goods_dict) return showcart() else: return "库存不足" else: return "数量必须为数字"
def showdir2(dir1, dir1_dict): """ 输出商品小类 :param goods_file:接收存有商品数据的文件 :param dir_1:接收一级目录列表 :param input_dir1:接收用户输入的一级目录 :return:返回商品二级目录 """ goods_dict = utils.load_file(GOODS_PATH) dir2 = str() if dir1_dict.get(dir1): dir2_info = goods_dict[dir1_dict[dir1]] else: return showdir1() dir2_dict = {} num = 1 for key in dir2_info: dir2 += "%s: %s " % (num, key) dir2_dict[str(num)] = key num += 1 chodir2 = input(dir2) if dir2_dict.get(chodir2): print(dir2_dict) return showgoods(dir1, chodir2, dir1_dict, dir2_dict) if chodir2 == "b": return showdir1()
def deposit(card_num, amount_of_money): """存款""" acc_path = os.path.join(USER_PATH, card_num, "account.json") acc = utils.load_file(acc_path) acc["deposit"] += amount_of_money utils.dump_to_file(acc_path, acc) system.recode_trade(card_num, "0", "0", amount_of_money)
def register(username, password): """ 用户注册 :param username:用户名 :param password:密码 :return:注册成功返回username,失败返回False """ patt1 = "[^(\w|_)]" patt2 = "^[^a-zA-Z]+" m = hashlib.md5() m.update(bytes(password, encoding='utf-8')) m_password = m.hexdigest() if re.search(patt1, username) or re.match(patt2, username): print("用户名只能为英文字母开头的数字、字母、下划线的组合!") return False else: acc = utils.load_file(ACC_PATH) if acc.get(username): print("用户名已存在!") return False else: acc[username] = {} acc[username]["username"] = username acc[username]["pwd"] = m_password acc[username]["balance"] = 0 acc[username]["locked_flag"] = 0 utils.dump_to_file(ACC_PATH, acc) return username
def setEditorTextCode(self, path): """设置加载合约的路径信息""" self._editor["path"] = path if os.path.isfile(path): self._editor["code"] = load_file(path) else: self._editor["code"] = ""
def billing_query(card_num, date): """账单查询""" bill_path = os.path.join(USER_PATH, card_num, "bill.json") bill = utils.load_file(bill_path) bill_data = bill[date] dept = bill_data["dept"] repayment = bill_data["repayment"] return dept, repayment
def lock(username): ad_path = os.path.join(DB_PATH, "credit", "admin", "admin.json") ad_info = utils.load_file(ad_path) if ad_info.get(username): ad_info[username]["lock_flag"] = 1 utils.dump_to_file(ad_path, ad_info) else: return "不存在此管理员"
def del_account(card_num): num_path = os.path.join(USER_PATH, card_num) acc_path = os.path.join(num_path, "account.json") # name_path = os.path.join(USER_PATH, "username.json") sdate_path = os.path.join(USER_PATH, "usersdate.json") acc = utils.load_file(acc_path) # name_list = utils.load_file(name_path) sdate = utils.load_file(sdate_path) balance = acc["credit_balance"] if balance < acc["credit_total"]: return "该用户有欠款未还清,不能注销!" else: acc["state"] = 2 for d in sdate: num_list = sdate[d] if card_num in num_list: num_list.remove(card_num) utils.dump_to_file(acc_path, acc) utils.dump_to_file(sdate_path, sdate)
def lock(username): num_path = os.path.join(DB_PATH, "credit", "users", username) if os.path.exists(num_path): acc_path = os.path.join(DB_PATH, "credit", "users", username, "account.json") else: return "账户不存在" acc_info = utils.load_file(acc_path) acc_info[username]["state"] = 1 utils.dump_to_file(acc_path, acc_info)
def load_preprocessed_data(self, *args): preprocess_dir = self.conf["paths"]["preprocessed_data"] data = [] for filename in args: data_ = u.load_file(Path(preprocess_dir) / filename) data.append(data_) if len(data) > 1: return data else: return data[0]
def transfer_accounts(to_num, amount_of_money): """转账""" global USER_INFO num = USER_INFO["username"] acc_path = os.path.join(USER_PATH, num) to_acc_path = os.path.join(USER_PATH, to_num) acc_path = os.path.join(acc_path, "account.json") to_acc_path = os.path.join(acc_path, "account.json") acc = utils.load_file(acc_path) try: to_acc = utils.load_file(to_acc_path) except Exception as e: print(e) if acc["deposit"] + acc["credit_balance"] >= amount_of_money: temp = to_acc["credit_balance"] + amount_of_money commission = 0 if acc["deposit"] >= amount_of_money: acc["deposit"] -= amount_of_money if temp <= to_acc["credit_total"]: to_acc["credit_balance"] += amount_of_money else: temp1 = temp - to_acc["credit_total"] to_acc["credit_balance"] = to_acc["credit_total"] to_acc["deposit"] += temp1 else: temp2 = amount_of_money - acc["deposit"] acc["deposit"] = 0 commission = temp2 * settings.FETCH_MONEY_RATE acc["credit_balance"] -= temp2 + commission if temp <= to_acc["credit_total"]: to_acc["credit_balance"] += amount_of_money else: temp3 = temp - to_acc["credit_total"] to_acc["credit_balance"] = to_acc["credit_total"] to_acc["deposit"] += temp3 utils.dump_to_file(acc_path, acc) utils.dump_to_file(to_acc_path, to_acc) system.recode_trade(num, "1", "1", amount_of_money, commission) system.recode_trade(to_num, "0", "0", amount_of_money) else: return "余额不足!"
def prepare_words(): """ Function to be called during server init. Loads the word to ID and ID to word dictionaries, in keeping with model vocabulary. """ global word_to_id,id_to_word if DEBUG >=2: print("Loading vocabulary from disk") vocab = utils.load_file(VOCAB_FILE) for index,v in enumerate(vocab): word_to_id[v] = index id_to_word[index] = v
def locking(username): """ 锁定用户函数 :param acc_file:账户文件 :param username:用户名 :return: """ global ACC_PATH acc = utils.load_file(ACC_PATH) user_info = acc[username] user_info["locked_flag"] = 1 utils.dump_to_file(ACC_PATH, acc)
def setEditorTextCode(self, path): """设置加载合约的路径信息""" try: if os.path.isfile(path): code = load_file(path) else: code = "" except Exception as e: self._logger.error("打开策略失败!!") traceback.print_exc(e) else: self._editor["path"] = path self._editor["code"] = code
def repayment(card_num, amount_of_money): """还款""" acc_path = os.path.join(USER_PATH, card_num, "account.json") acc = utils.load_file(acc_path) amounts_owed = acc["credit_total"] - acc["credit_balance"] if amount_of_money >= amounts_owed: acc["credit_balance"] = acc["credit_total"] acc[deposit] += (amount_of_money - amounts_owed) utils.dump_to_file(acc_path, acc) else: acc["credit_balance"] += amount_of_money utils.dump_to_file(acc_path, acc) system.recode_trade(card_num, "0", "0", amount_of_money)
def modify_account(card_num, new_total=None, new_date=None, new_state=None): acc_path = os.path.join(USER_PATH, card_num, "account.json") acc = utils.load_file(acc_path) if new_total: temp = acc["credit_total"] ran = new_total - temp acc["credit_total"] = new_total acc["credit_balance"] += ran if new_date: acc["STATEMENT_DATE"] = new_date if new_state: acc["state"] = new_state utils.dump_to_file(acc_path)
def init_train(self, train_path, data_path="store/train_data.pkl"): """初始化数据""" try: print("开始载入初始化数据....", file=sys.stderr) train_data = load_file(data_path) except BaseException: train_data = dict() for line in process_data.read_file(filename=train_path): arr = line.split('\t') train_data.setdefault(int(arr[0]), set()) train_data[int(arr[0])].add(int(arr[1])) save_file(data_path, train_data) return train_data
def train(self, sim_matrix_path="store/buy_user_sim.pkl"): print("开始训练模型", file=sys.stderr) try: print("开始载入用户协同矩阵....", file=sys.stderr) self.user_sim_matrix = load_file(sim_matrix_path) print("载入协同过滤矩阵完成", file=sys.stderr) except BaseException: print("载入用户协同过滤矩阵失败,重新计算协同过滤矩阵", file=sys.stderr) # 计算用户协同矩阵 self.user_sim_matrix = self.user_similarity() print("开始保存协同过滤矩阵", file=sys.stderr) save_file(sim_matrix_path, self.user_sim_matrix) print("保存协同过滤矩阵完成", file=sys.stderr)
def set_estimator_params(estimator, conf_model, exp_dir): verbose = conf_model["verbose"] path_best_params = exp_dir / "best_params.json" if (len(conf_model["params"]) != 0) and (not conf_model["search_best_params"]): u.vprint("Using param from conf file : \n %s" % (str(conf_model["params"])), verbose) estimator.set_params(**conf_model["params"]) elif path_best_params.exists(): u.vprint("Loading best params", verbose) best_params = u.load_file(exp_dir / "best_params.json") u.vprint(str(best_params), verbose) estimator.set_params(**best_params) else: # Default parameters u.vprint("Using default params", verbose) u.vprint(str(estimator.get_params()), verbose) return estimator
def spend(card_num, card_pwd, amount_of_money): """消费""" acc_path = os.path.join(USER_PATH, card_num, "account.json") acc = utils.load_file(acc_path) if acc["state"] == 0: if card_pwd != acc["pwd"]: return "支付密码不正确" else: if acc["credit_balance"] >= amount_of_money: acc["credit_balance"] -= amount_of_money utils.dump_to_file(acc_path, acc) system.recode_trade(card_num, "1", "1", amount_of_money) else: return "额度不足" else: return "您的账户已冻结"
def login(username, password): """用户登录 :param acc_file:账户文件 :param username:用户名 :param password:密码 :return:成功-True,密码不对-False,用户不存在-None,已锁定-"locked" """ password = utils.encrypt(password) acc = utils.load_file(ACC_PATH) if not acc.get(username): return "用户名不存在!" else: if username == acc[username]["username"] and\ password == acc[username]["pwd"]: return acc[username] else: return "用户名或密码不正确"
def recode_trade(card_num, trade_flag, opera_type, amount, commission=0): """记录操作流水""" num_path = os.path.join(USER_PATH, card_num) trade_path = os.path.join(num_path, "trade.json") timenow = arrow.now().strftime("%Y-%m-%d %H:%M:%S") if os.path.exists(trade_path): trade = utils.load_file(trade_path) else: trade = dict() # print(trade) trade[timenow] = dict() trade[timenow]["trade_flag"] = trade_flag trade[timenow]["opera_type"] = opera_type trade[timenow]["amount"] = amount trade[timenow]["commission"] = commission print(trade_path) utils.dump_to_file(trade_path, trade)
def eval_model_text_level(conf_model, estimator, X_train, X_test, conf=None): adjust_with_nanc_classifier = conf_model["adjust_with_nanc_classifer"] path_model_df_prep = Path(conf_model["path"] + "df_train_preprocessed.csv") df = u.load_file(path_model_df_prep, usecols=["ID", "date_accident", "date_consolidation"]) df = df.rename(columns={"ID": "txt_id"}) df_y_pred_test = predict_text_level(conf_model, estimator, X_test, adjust_with_nanc_classifier, conf) df_y_test = df[df["txt_id"].isin(df_y_pred_test["txt_id"])].drop_duplicates().copy() acc1_test, acc2_test, mean_acc_test = compute_accuracy(df_y_test, df_y_pred_test) df_y_pred_train = predict_text_level(conf_model, estimator, X_train, adjust_with_nanc_classifier, conf) df_y_train = df[df["txt_id"].isin(df_y_pred_train["txt_id"])].drop_duplicates().copy() acc1_train, acc2_train, mean_acc_train = compute_accuracy(df_y_train, df_y_pred_train) res3 = "Train : accuracy = %.2f (accident = %.2f, consolidation = %.2f)\n" \ "Test : accuracy = %.2f (accident = %.2f, consolidation = %.2f)" % \ (mean_acc_train, acc1_train, acc2_train, mean_acc_test, acc1_test, acc2_test) return res3
def showdir1(): """ 输出商品一级目录 :param goods_file:接收存有商品数据的文件 :return: 返回商品的一级目录 """ goods_dict = utils.load_file(GOODS_PATH) dir1 = str() num = 1 dir1_dict = {} for key in goods_dict: dir1 += "%s: %s " % (num, key) dir1_dict[str(num)] = key num += 1 chodir1 = input(dir1) if chodir1 == "b": return sp.main() return showdir2(chodir1, dir1_dict)