def seg_flag_cor_batch(path_in, path_out, names):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("processing file: {}".format(file))
data = pd.read_csv(path_in + file)
data = seg_flag_cor(data, names)
data.to_csv(path_out + file, index=False)
def interpolate_bin_mean_batch(path_in, path_out, bin_size=5):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("processing file: {}".format(file))
data = pd.read_csv(path_in + file)
data = interpolate_bin_mean(data, bin_size)
data.to_csv(path_out + file, index=False)
def del_dup_batch(path_in, path_out):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("processing file: {}".format(file))
data = pd.read_csv(path_in+file)
data_new = del_dup(data)
data_new.to_csv(path_out+file, index=False)
def seg_flag_cor_batch(path_in, path_out, names):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("processing file: {}".format(file))
data = pd.read_csv(path_in+file)
data = seg_flag_cor(data, names)
data.to_csv(path_out+file, index=False)
def del_dup_batch(path_in, path_out):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("processing file: {}".format(file))
data = pd.read_csv(path_in + file)
data_new = del_dup(data)
data_new.to_csv(path_out + file, index=False)
def sm_mean_1m_batch(path_in, path_out):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("-------- processing file: {}".format(file))
data = pd.read_csv(path_in + file)
data = sm_mean_1m(data)
data.to_csv(path_out + file, index=False)
def cal_rolling_path(path_in, path_out, func, win_rate):
filist = base.get_files_csv(path_in)
for file in filist:
print("caculate for file {}".format(file))
cal_rolling_stats(file_in=path_in + file,
file_out=path_out + file,
func=func,
win_rate=win_rate)
def cal_rolling_path(path_in, path_out, func, win_rate):
filist =base.get_files_csv(path_in)
for file in filist:
print("caculate for file {}".format(file))
cal_rolling_stats(file_in=path_in + file,
file_out=path_out + file,
func=func,
win_rate=win_rate)
def plot_box_ZX_HW_batch(path):
filelist = base.get_files_csv(path)
name = var_name.name_HW1 + var_name.name_HW2
for file in filelist:
plt.figure(figsize=(13, 2))
file_path = path + file
data = pd.read_csv(file_path)
plt.boxplot(data[name].as_matrix())
plt.xticks(np.arange(1, 1 + len(name)), name)
plt.title(file)
def plot_box_ZX_HW_batch(path):
filelist = base.get_files_csv(path)
name = var_name.name_HW1 + var_name.name_HW2
for file in filelist:
plt.figure(figsize=(13, 2))
file_path = path + file
data = pd.read_csv(file_path)
plt.boxplot(data[name].as_matrix())
plt.xticks(np.arange(1, 1+len(name)), name)
plt.title(file)
def parse_time_batch(path_in, path_out):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("process file: {}".format(file))
data = pd.read_csv(path_in + file)
data['BTSJ_I'] = parse_time(list(data['BTSJ']))
names = list(data.columns)
names.remove('BTSJ_I')
names.insert(1, 'BTSJ_I')
data = data[names]
data.to_csv(path_out + file, index=False)
def parse_temp_batch(path_in, path_out):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("processing file {}".format(file))
data = pd.read_csv(path_in+file)
if len(data) < 1000: # discard the dataset with too less records
print("!!!!!! without parsing file:{} because the number of records too small!".format(file))
continue
data = parse_temperature(data)
data = parse_temp_HW(data)
data.to_csv(path_out+file, index=False)
def parse_time_batch(path_in, path_out):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("process file: {}".format(file))
data = pd.read_csv(path_in+file)
data['BTSJ_I'] = parse_time(list(data['BTSJ']))
names = list(data.columns)
names.remove('BTSJ_I')
names.insert(1, 'BTSJ_I')
data = data[names]
data.to_csv(path_out+file, index=False)
def plot_stats_ZX_HW_batch(path, func):
plt.figure(figsize=(15, 5))
name = var_name.name_HW1 + var_name.name_HW2
filelist = base.get_files_csv(path)
for file in filelist:
file_path = path + file
data = pd.read_csv(file_path)
stds = data[name].apply(func)
plt.plot(stds.values, 'o-', label=file)
plt.xticks(np.arange(stds.count()), stds.index)
plt.legend(loc='best', prop={'size': 8})
plt.title("std of ZH_HW")
def check_bug_batch(path_in):
filelist = base.get_files_csv(path_in)
f_log = open(path_in+"check_bug_log.txt", "w")
stdout_origin = sys.stdout
sys.stdout = f_log
for file in filelist:
print("--------------- file: {}".format(file))
check_bug(path_in+file)
print("---------------\n\n")
f_log.close()
sys.stdout = stdout_origin
def check_seg_flag_batch(path, log_dir):
filelist = base.get_files_csv(path)
stdout_origin = sys.stdout
f_log = open(log_dir, "w")
sys.stdout = f_log
for file in filelist:
print("**************** file:{} ***************".format(file))
check_seg_flag(path+file)
print("****************************************\t")
sys.stdout = stdout_origin
f_log.close()
def check_seg_flag_batch(path, log_dir):
filelist = base.get_files_csv(path)
stdout_origin = sys.stdout
f_log = open(log_dir, "w")
sys.stdout = f_log
for file in filelist:
print("**************** file:{} ***************".format(file))
check_seg_flag(path + file)
print("****************************************\t")
sys.stdout = stdout_origin
f_log.close()
def parse_temp_batch(path_in, path_out):
filelist = base.get_files_csv(path_in)
for file in filelist:
print("processing file {}".format(file))
data = pd.read_csv(path_in + file)
if len(data) < 1000: # discard the dataset with too less records
print(
"!!!!!! without parsing file:{} because the number of records too small!"
.format(file))
continue
data = parse_temperature(data)
data = parse_temp_HW(data)
data.to_csv(path_out + file, index=False)
def plot_stats_ZX_HW_batch(path, func):
plt.figure(figsize=(15, 5))
name = var_name.name_HW1 + var_name.name_HW2
filelist = base.get_files_csv(path)
for file in filelist:
file_path = path + file
data = pd.read_csv(file_path)
stds = data[name].apply(func)
plt.plot(stds.values, 'o-', label=file)
plt.xticks(np.arange(stds.count()), stds.index)
plt.legend(loc='best', prop={'size': 8})
plt.title("std of ZH_HW")
def check_bug_batch(path_in):
filelist = base.get_files_csv(path_in)
f_log = open(path_in + "check_bug_log.txt", "w")
stdout_origin = sys.stdout
sys.stdout = f_log
for file in filelist:
print("--------------- file: {}".format(file))
check_bug(path_in + file)
print("---------------\n\n")
f_log.close()
sys.stdout = stdout_origin
def cluster_batch(data_dir):
filelist = base.get_files_csv(data_dir)
stdout_original = sys.stdout
log_file = open(data_dir+"dtw_cluster.txt", "w")
sys.stdout = log_file
for file in filelist:
data = pd.read_csv(data_dir + file)
del data['BTSJ']
#data = (data-data.mean())/(data.std())
clust = cluster(data)
print("-------------------------------------------------------")
print("********process file: {}*********".format(file))
print(clust)
sys.stdout = stdout_original
log_file.close()
def cluster_batch(data_dir):
filelist = base.get_files_csv(data_dir)
stdout_original = sys.stdout
log_file = open(data_dir + "dtw_cluster.txt", "w")
sys.stdout = log_file
for file in filelist:
data = pd.read_csv(data_dir + file)
del data['BTSJ']
#data = (data-data.mean())/(data.std())
clust = cluster(data)
print("-------------------------------------------------------")
print("********process file: {}*********".format(file))
print(clust)
sys.stdout = stdout_original
log_file.close()
import pandas as pd
import os
from preprocess import base
from ffx_learn import learn_ffx
from ffx_learn import call_ffx
def norm_2(x):
return (x - np.mean(x)) / (x.max() - x.min())
#return (x-x.mean()) / np.sqrt(x*x)
root_dir = os.getcwd()
data_dir = root_dir + "/data_0134/integrated_temp_sdmc_mean/"
filelist = base.get_files_csv(data_dir)
print(filelist)
file = filelist[0]
data = pd.read_csv(data_dir + file)
del data['BTSJ']
print(data.columns)
#data = data.apply(norm_2, axis=1)
# ax = data.plot()
# ax.set_title(file)
# #ax.set_ylim([-1, 1])
# plt.show()
# varnames = data.columns
# X = data.as_matrix()
# y = np.zeros(X.shape[0])
import numpy as np
import pandas as pd
import os
from preprocess import base
from ffx_learn import learn_ffx
from ffx_learn import call_ffx
def norm_2(x):
return (x-np.mean(x))/(x.max()-x.min())
#return (x-x.mean()) / np.sqrt(x*x)
root_dir = os.getcwd()
data_dir = root_dir + "/data_0134/integrated_temp_sdmc_mean/"
filelist = base.get_files_csv(data_dir)
print(filelist)
file = filelist[0]
data = pd.read_csv(data_dir+file)
del data['BTSJ']
print(data.columns)
#data = data.apply(norm_2, axis=1)
# ax = data.plot()
# ax.set_title(file)
# #ax.set_ylim([-1, 1])
# plt.show()
# varnames = data.columns
# X = data.as_matrix()
# y = np.zeros(X.shape[0])