def extract_selected_id_name(): """ :return: the id name of the selected pics in a file """ root_dir = get_user_data_dir() original_path = join( root_dir, "706_dnm_tmp_3ZFwT#sum_iUserNum_300#20190620_16Days_valid6D_pipeline_test_input_node_train_data.csv" ) df = pd.read_csv(original_path) # print(df.head(2)) # print df.columns.tolist() df["timestamp"] = df["timestamp"].map(millisec_to_str) # pic_path = join("/Users/xumiaochun/jiawei", "tmp/pic_valid_select/") file_dir = pic_path i = 1 a = os.walk(file_dir) b = None for root, dirs, files in os.walk(file_dir): print(i) i += 1 print(root) #当前目录路径 print(dirs) #当前路径下所有子目录 print(files) #当前路径下所有非目录子文件 return b
def gen_pic(): """ :return: the plots of the multiple id dataset based on the final date """ root_dir = get_user_data_dir() original_path = join( root_dir, "706_dnm_tmp_3ZFwT#sum_iUserNum_300#20190620_16Days_valid6D_pipeline_test_input_node_train_data.csv" ) df = pd.read_csv(original_path) df["timestamps"] = df["timestamps"].map(millisec_to_str) # pic_path = join("/Users/xumiaochun/jiawei", "tmp/pic_valid/result_train/") line_id_list = np.unique(df.line_id) for l_id in line_id_list: l_id_list = l_id.split("valid") VALID_DAY = l_id_list[-1].replace("D", "") # print (VALID_DAY) if int(VALID_DAY) < 10: continue df_slice = df[df.line_id == l_id].copy() print(df_slice.shape) plt = plot_hist(df_slice, detect_days=2, plot_day_index=[1, 7], anom_col="label", value_col="point", freq=300) savePNG(plt, targetDir=join(pic_path, "%s.png" % l_id))
# -*- coding: utf-8 -*- # Standard library from os.path import join from os import listdir # Third Party Library import pandas as pd import numpy as np # My Library from common.path_helper import split_dir, saveDF from settings import Config_json, get_user_data_dir from step1_dataIO import load_level_df config_json = Config_json() root_dir = get_user_data_dir() input_dir = join(root_dir, config_json.get_config("original_data")) output_dir = join(root_dir, config_json.get_config("STEP1_DATA_SUBDIR")) #global parameter for model DELTA_T2_POINTS = config_json.get_config("DELTA_T2_POINTS") DELTA_T1_POINTS = config_json.get_config("DELTA_T1_POINTS") ###make spread_crossing labels delta_Events = config_json.get_config("PREDICT_LENGTH_POINTS") def spread_crossing(a): """ :param P1: price object contains bid/ask :param P2: price object contains bid/ask :return: spread_crossing of p1, p2,i.e,p2 - p1 """ P1_ask, P1_bid, P2_ask, P2_bid = a