def formation(filename1,filename2): # it's the function which make file2 same format of file1
if (filename1[0] == "/"):
filename1 = filename1
elif ((filename1[0] == "C") & (filename1[1] == ":")):
filename1 = filename1
else:
filename1 = os.getcwd() + "/" + filename1 # get the path included filename
loca1 = len(filename1)
for i in range(1, len(filename1) + 1): # find the "/" location
if (filename1[-i] == "/"):
loca1 = i - 1
break
FILENAME1 = filename1.replace(filename1[:-loca1], "") # this is the shorten filename
filename1_No_Txt = FILENAME1.replace(".txt", "")
infile1 = filename1
if (filename2[0] == "/"):
filename2 = filename2
elif ((filename2[0] == "C") & (filename2[1] == ":")):
filename2 = filename2
else:
filename2 = os.getcwd() + "/" + filename2 # get the path included filename
loca2 = len(filename2)
for i in range(1, len(filename2) + 1): # find the "/" location
if (filename2[-i] == "/"):
loca2 = i - 1
break
FILENAME2 = filename2.replace(filename2[:-loca2], "") # this is the shorten filename
filename2_No_Txt = FILENAME2.replace(".txt", "")
infile2 = filename2
DATE_ROW = MakeList_column(infile1)
DATA_ROW = MakeList_column(infile2)
'''
def DATA_Merger(infilename1, infilename2):
filename_list1 = read_file_name(infilename1)
infile1 = filename_list1[2]
filename_list2 = read_file_name(infilename2)
infile2 = filename_list2[2]
DATE_ROW = MakeList_column(infile1)
DATA_ROW = MakeList_column(infile2)
for i in range(len(DATA_ROW)):
DATE_ROW.append(DATA_ROW[i])
NEWNAME = infile1.replace(".txt", "_MERGE_" + filename_list2[0])
FN = NEWNAME + ".txt"
Of = open(FN, "w+")
for j in range(len(DATE_ROW[0])):
for i in range(len(DATE_ROW)):
Of.write("%s" % DATE_ROW[i][j])
if (i != len(DATE_ROW) - 1):
Of.write(" ")
else:
Of.write("\n")
# Of.write("%s " %DATE_ROW[0][j])
# Of.write("%s\n" %DATE_ROW[1][j])
Of.close()
return FN
def Merge(filename1, filename2):
if(filename1[0]=="/"):
filename1 = filename1
elif((filename1[0]=="C")&(filename1[1]==":")):
filename1 = filename1
else:
filename1 = os.getcwd() + "/" + filename1 # get the path included filename
loca1=len(filename1)
for i in range (1,len(filename1)+1): # find the "/" location
if(filename1[-i] == "/"):
loca1 = i-1
break
FILENAME1 = filename1.replace(filename1[:-loca1],"") # this is the shorten filename
filename1_No_Txt = FILENAME1.replace(".txt","")
infile1 = filename1
if(filename2[0]=="/"):
filename2 = filename2
elif((filename2[0]=="C")&(filename2[1]==":")):
filename2 = filename2
else:
filename2 = os.getcwd() + "/" + filename2 # get the path included filename
loca2=len(filename2)
for i in range (1,len(filename2)+1): # find the "/" location
if(filename2[-i] == "/"):
loca2 = i-1
break
FILENAME2 = filename2.replace(filename2[:-loca2],"") # this is the shorten filename
filename2_No_Txt = FILENAME2.replace(".txt","")
infile2 = filename2
DATE_ROW = MakeList_column(infile1)
DATA_ROW = MakeList_column(infile2)
# DATE_ROW.append(DATA_ROW[0])
for i in range(len(DATA_ROW)):
DATE_ROW.append(DATA_ROW[i])
# print(DATE_ROW)
# print(len(DATE_ROW[0]))
# print(len(DATE_ROW[1]))
NEWNAME = infile1.replace(".txt","_MERGE_"+filename2_No_Txt)
FN = NEWNAME+".txt"
Of = open(FN,"w+")
for j in range(len(DATE_ROW[0])):
for i in range(len(DATE_ROW)):
if DATE_ROW[i][j] is None:
continue
else:
Of.write("%s" %DATE_ROW[i][j])
if(i != len(DATE_ROW)-1):
Of.write(" ")
else:
Of.write("\n")
# Of.write("%s " %DATE_ROW[0][j])
# Of.write("%s\n" %DATE_ROW[1][j])
Of.close()
return FN
def rawdata_histogram(filename, BIN=8):
histo_list = MakeList_column(filename)
for ii in range(len(histo_list)):
if ii == 0: ##pass 'DATE' list like as 201312,201401 ...
continue
else:
list_A = (histo_list[ii])
list_float = []
for i in range(len(list_A)):
if i == 0:
continue
else:
list_float.append(float(list_A[i]))
# print(list_float)
title = (str(list_A[0]))
AVG = sum(list_float, 0.0) / len(list_float)
# print (AVG)
AVG = ("%.3f" % round(AVG, 3))
plt.hist(list_float, bins=BIN, label='mean = ' + (str(AVG)))
plt.title(title)
plt.ylabel('Frequency')
plt.xlabel(title + '__score')
plt.legend()
plt.show()
def Month_divide(filename):
FILE = read_file_name(filename)
infile1 = FILE[2]
OUTPUT_PATH = FILE[3] + FILE[0]
COL_LIST1 = MakeList_column(infile1)
#print(COL_LIST1)
M_LIST = []
for k in range(len(COL_LIST1[0])):
gg = []
for m in range(len(COL_LIST1)):
gg.append(COL_LIST1[m][k])
M_LIST.append(gg)
# if k==0:
# continue
#print(M_LIST)
RETURN_LIST = []
bb = []
for a in range(len(M_LIST)):
# aa=[]
# aa.append(M_LIST[a])
if a > 1 and (float(M_LIST[a][0]) - float(M_LIST[a - 1][0])) > 1:
#print(bb)
FN = "M" + str(int(M_LIST[a - 1][0]) // 100) + ".txt"
FN = OUTPUT_PATH + "_" + FN
RETURN_LIST.append(FN)
Of = open(FN, "w+")
for i in range(len(bb)):
for j in range(len(bb[i])):
Of.write("%s" % bb[i][j])
if (j != len(bb[i]) - 1):
Of.write(" ")
if (j == len(bb[i]) - 1):
Of.write("\n")
Of.close()
bb = []
bb.append(M_LIST[0])
cc = bb
bb.append(M_LIST[a])
# print(bb)
#print(bb)
#print(cc)
FN = "M" + str(int(cc[1][0]) // 100) + ".txt"
FN = OUTPUT_PATH + "_" + FN
Of = open(FN, "w+")
for i in range(len(cc)):
for j in range(len(cc[i])):
Of.write("%s" % cc[i][j])
if (j != len(cc[i]) - 1):
Of.write(" ")
if (j == len(cc[i]) - 1):
Of.write("\n")
Of.close()
RETURN_LIST.append(FN)
return RETURN_LIST
def Converting_largeBin(filename, NBINS=10000):
if(filename[0]=="/"):
filename = filename
elif((filename[0]=="C")&(filename[1]==":")):
filename = filename
else:
filename = os.getcwd() + "/" + filename # get the path included filename
loca=len(filename)
for i in range (1,len(filename)+1): # find the "/" location
if(filename[-i] == "/"):
loca = i-1
break
FILENAME = filename.replace(filename[:-loca],"") # this is the shorten filename
filename_No_Txt = FILENAME.replace(".txt","")
infile = filename
OUPUT_TXT_LIST = []
Col_List = MakeList_column(infile)
for i in range(len(Col_List)):
HIST_RANGE = min_max_range(Col_List[i])
print(HIST_RANGE)
SAVENAME = os.getcwd()+"/"+filename_No_Txt+"_"+Col_List[i][0]+"_largeBin_py.txt"; #print(SAVENAME)
ff = open(SAVENAME,"w+")
OUPUT_TXT_LIST.append(SAVENAME)
FLAG = (HIST_RANGE[3] - HIST_RANGE[2])/float(NBINS)
for j in range(NBINS):
COUNT_NUM = Counting(Col_List[i],[HIST_RANGE[2]+FLAG*(j),HIST_RANGE[2]+FLAG*(j+1)])
ff.write("%i %f %f %f\n" %(j+1,HIST_RANGE[2]+FLAG*(j),HIST_RANGE[2]+FLAG*(j+1),COUNT_NUM))
ff.close()
return OUPUT_TXT_LIST
def main():
AQI = AQI_CALC()
AQI.AWAKE_BROWSER()
AQI.Access_URL(URL="http://aqicn.org/calculator/kr/")
#AQI.Click_for_DATA()
col_list = MakeList_column(
"/Users/leejunho/Desktop/git/python3Env/group_study/NOT_USUALLY_VISIT/statistic_group_study/R_language/test/SEOUL_Day_R_P.txt"
)
Write_list = []
for i in range(1, 2):
if i == 0:
print("choose PM2.5")
OUT_NAME = "PM2p5"
elif i == 1:
print("choose PM10")
OUT_NAME = "PM10"
elif i == 2:
print("choose O3(8hr)")
OUT_NAME = "O3"
elif i == 3:
print("choose SO2(1hr)")
OUT_NAME = "SO2"
elif i == 4:
print("choose NO2")
OUT_NAME = "NO2"
elif i == 5:
print("choose CO")
OUT_NAME = "CO"
AQI.Click_for_DATA()
temp_list = []
for j in range(len(col_list[i])):
# for j in range(10):
if (j == 0):
continue
AQI.Input_value(col_list[i][j])
DATA_out = AQI.Output_DATA()
DATA_out = float(DATA_out)
print(j, DATA_out)
temp_list.append(DATA_out)
Write_list.append(temp_list)
outfile = "/Users/leejunho/Desktop/git/python3Env/group_study/NOT_USUALLY_VISIT/statistic_group_study/R_language/test/AQI_PM10.txt"
AQI.CREATE_n_WRITE_INTO_TXT(outfileName=outfile, Write_LIST=Write_list)
# AQI.Input_value('100')
AQI.Output_DATA()
def missing_value(filename1):
if (filename1[0] == "/"):
filename1 = filename1
elif ((filename1[0] == "C") & (filename1[1] == ":")):
filename1 = filename1
else:
filename1 = os.getcwd(
) + "/" + filename1 # get the path included filename
loca1 = len(filename1)
for i in range(1, len(filename1) + 1): # find the "/" location
if (filename1[-i] == "/"):
loca1 = i - 1
break
FILENAME1 = filename1.replace(filename1[:-loca1],
"") # this is the shorten filename
filename1_No_Txt = FILENAME1.replace(".txt", "")
infile1 = filename1
DATA_ROW = MakeList_column(infile1)
arr_row = np.array(DATA_ROW)
i, j = 0, 0
print(len(arr_row), len(arr_row[0]))
while i <= (len(arr_row) - 1):
while j <= (len(arr_row[0]) - 1):
if arr_row[i][j] == 'None':
# print(1)
arr_row = np.delete(arr_row, j, axis=1)
else:
j += 1
i += 1
j = 0
NEWNAME = infile1.replace(filename1_No_Txt + ".txt",
"missing_deleted_" + filename1_No_Txt)
FN = NEWNAME + ".txt"
Of = open(FN, "w+")
for j in range(len(arr_row[0])):
for i in range(len(arr_row)):
if i == (len(arr_row) - 1):
Of.write(arr_row[i][j] + "\n")
else:
Of.write(arr_row[i][j] + " ")
Of.close()
return FN
def line_graph(filename, y_axis):
Raw_list = MakeList_column(filename)
Graph_list_subject = []
Graph_list_body = []
for i in range(len(Raw_list)):
Temp_list_subject = []
Temp_list_body = []
for k in range(len(Raw_list[i])):
if k == 0:
Temp_list_subject.append(Raw_list[i][k])
else:
Temp_list_body.append(float(Raw_list[i][k]))
Graph_list_subject.append(Temp_list_subject)
Graph_list_body.append(Temp_list_body)
plt.plot(Graph_list_body[y_axis])
plt.title(Graph_list_subject[y_axis])
plt.show()
def Post_hoc(infile_list, alpha=0.05):
#http://cleverowl.uk/2015/07/01/using-one-way-anova-and-tukeys-test-to-compare-data-sets/
#https://stackoverflow.com/questions/16049552/what-statistics-module-for-python-supports-one-way-anova-with-post-hoc-tests-tu
#alpha : The significance level is the probability of rejecting the null hypothesis when it is true
#P-values are the probability of obtaining an effect at least as extreme as the one in your sample data, assuming the truth of the null hypothesis.
ANOVA_list = []
for i in range(len(infile_list)):
temp_list = MakeList_column(infile_list[i])[1]
temp_list2 = []
for j in range(len(temp_list)):
if j == 0:
continue
temp_list2.append(float(temp_list[j]))
#print(temp_list2)
ANOVA_list.append(temp_list2)
Name_list = []
list_1996 = ANOVA_list[0]
#name_1996= []
for i in range(len(list_1996)):
Name_list.append("1996-2000")
list_2001 = ANOVA_list[1]
#name_2001= []
for i in range(len(list_2001)):
Name_list.append("2001-2005")
list_2006 = ANOVA_list[2]
#name_2006= []
for i in range(len(list_2006)):
Name_list.append("2006-2011")
list_2012 = ANOVA_list[3]
#name_2012= []
for i in range(len(list_2012)):
Name_list.append("2012-2017")
Num_list = []
for i in range(len(ANOVA_list)):
for j in range(len(ANOVA_list[i])):
Num_list.append(ANOVA_list[i][j])
#print(len(Name_list),len(Num_list))
mc = MultiComparison(Num_list, Name_list)
result = mc.tukeyhsd(alpha)
print(result)
print(mc.groupsunique)
return result
def Daily_data(filename):
FILE = read_file_name(filename)
infile1 = FILE[2]
COL_LIST1 = MakeList_column(infile1)
# print(COL_LIST1)
for s in range(len(COL_LIST1)-1):
# print(COL_LIST1[s][0])
x= [COL_LIST1[0][i+1] for i in range(len(COL_LIST1[0])-1)]
y = [float(COL_LIST1[s+1][j+1]) for j in range(len(COL_LIST1[0])-1)]
plt.xlabel(COL_LIST1[0][0])
plt.ylabel(COL_LIST1[s+1][0])
# print(x,y)
plt.plot(x,y,color='green',marker='o',linestyle='solid')
plt.grid(True)
SavefileName = FILE[0] + "_" + COL_LIST1[s][0] + "_month_plot.pdf"
print(SavefileName)
# plt.show()
plt.savefig(SavefileName)
plt.close('all')
def ANOVA(infile_list):
#https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.f_oneway.html
#print("test")
ANOVA_list = []
for i in range(len(infile_list)):
temp_list = MakeList_column(infile_list[i])[1]
temp_list2 = []
for j in range(len(temp_list)):
if j == 0:
continue
temp_list2.append(float(temp_list[j]))
#print(temp_list2)
ANOVA_list.append(temp_list2)
#print(len(ANOVA_list))
Free_between = len(ANOVA_list) - 1
Free_within = 0
for i in range(len(ANOVA_list)):
Free_within = Free_within + (len(ANOVA_list[i]) - 1)
RES = stats.f_oneway(ANOVA_list[0], ANOVA_list[1], ANOVA_list[2],
ANOVA_list[3])
#print(RES)
return [RES, Free_between, Free_within]
def box_plot(filename):
filename = filename
Raw_list = MakeList_column(filename)
Graph_list_subject = []
Graph_list_body = []
x_axis = 0
y_axis = 0
# 데이터를 이름과 값의 리스트로 분리합니다. 이름: Graph_list_subject, 값: Graph_list_value
for i in range(len(Raw_list)):
Temp_list_subject = []
Temp_list_body = []
for k in range(len(Raw_list[i])):
if k == 0:
Temp_list_subject.append(Raw_list[i][k])
else:
Temp_list_body.append(float(Raw_list[i][k]))
# 그래프용 리스트 완성
Graph_list_subject.append(Temp_list_subject)
Graph_list_body.append(Temp_list_body)
labels = [
Graph_list_subject[i + 1][0]
for i in range(len(Graph_list_subject[1:]))
]
fig1, ax = plt.subplots()
ax.set_title(filename)
ax.boxplot(Graph_list_body[1:],
vert=True,
labels=labels,
showmeans=True,
meanline=True,
whis=[0, 100])
ax.xaxis.set_tick_params(rotation=30, labelsize=10)
ax.yaxis.grid(True)
plt.show()
def scattergram(filename, x_axis, y_axis):
filename = filename
Raw_list = MakeList_column(filename)
Graph_list_subject = []
Graph_list_body = []
x_axis = 0
y_axis = 0
# 데이터를 이름과 값의 리스트로 분리합니다. 이름: Graph_list_subject, 값: Graph_list_value
for i in range(len(Raw_list)):
Temp_list_subject = []
Temp_list_body = []
for k in range(len(Raw_list[i])):
if k == 0:
Temp_list_subject.append(Raw_list[i][k])
else:
Temp_list_body.append(float(Raw_list[i][k]))
# 그래프용 리스트 완성
Graph_list_subject.append(Temp_list_subject)
Graph_list_body.append(Temp_list_body)
# 보고 싶은 x축, y축 데이터를 선택합니다.
for n in range(len(Graph_list_subject)):
print(n, ':', Graph_list_subject[n][0])
x_axis = int(input("x축에 해당하는 번호를 입력하세요:"))
y_axis = int(input("y축에 해당하는 번호를 입력하세요:"))
# 산포도 그래프 그리기
x_label = Graph_list_subject[x_axis][0]
y_label = Graph_list_subject[y_axis][0]
plt.plot(Graph_list_body[x_axis], Graph_list_body[y_axis], 'o')
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(x_label + '-' + y_label)
plt.show()
def N_sigma_skimming(filename, N_Of_Sigma=5):
if (filename[0] == "/"):
filename = filename
elif ((filename[0] == "C") & (filename[1] == ":")):
filename = filename
else:
filename = os.getcwd(
) + "/" + filename # get the path included filename
loca = len(filename)
for i in range(1, len(filename) + 1): # find the "/" location
if (filename[-i] == "/"):
loca = i - 1
break
FILENAME = filename.replace(filename[:-loca],
"") # this is the shorten filename
filename_No_Txt = FILENAME.replace(".txt", "")
infile = filename
ROW_list = MakeList(infile)
COL_list = MakeList_column(infile)
Mean_list = list()
Std_list = list()
# VAR_list = list()
Name_list = list()
for i in range(len(COL_list)):
Name_list.append(COL_list[i][0])
Mean_list.append(d0_exclude_1_mean((COL_list[i])))
# VAR_list.append(d0_exclude_1_variance((COL_list[i])))
Std_list.append(d0_exclude_1_STD((COL_list[i])))
# print(Name_list)
# print(Mean_list)
'''
RE_COL_LIST = []
for i in range(len(COL_list)):
if((COL_list[i][0] == DATE) | (COL_list[i][0] == HOLI) | (COL_list[i][0] == DAYS)):
continue
L_NSIGMA = Mean_list[i] - N_Of_Sigma*Std_list[i]
H_NSIGMA = Mean_list[i] + N_Of_Sigma*Std_list[i]
for j in range(len(COL_list[i])):
'''
ttt = 0
RE_ROW_LIST = []
for i in range(len(ROW_list)):
if (i == 0):
RE_ROW_LIST.append(ROW_list[i])
continue
for j in range(len(ROW_list[i])):
if (((Name_list[j] == "DATE") | (Name_list[j] == "HOLI") |
(Name_list[j] == "DAYS")) & (j != (len(ROW_list[i]) - 1))):
continue
L_NSIGMA = Mean_list[j] - N_Of_Sigma * Std_list[j]
H_NSIGMA = Mean_list[j] + N_Of_Sigma * Std_list[j]
# print(L_NSIGMA, H_NSIGMA)
if ((j == (len(ROW_list[i]) - 1)) &
((Name_list[j] == "DATE") | (Name_list[j] == "HOLI") |
(Name_list[j] == "DAYS"))):
RE_ROW_LIST.append(ROW_list[i])
elif ((float(ROW_list[i][j]) > L_NSIGMA) &
(float(ROW_list[i][j]) < H_NSIGMA)):
if (j == (len(ROW_list[i]) - 1)):
RE_ROW_LIST.append(ROW_list[i])
else:
continue
else:
ttt = ttt + 1
break
# print(ttt)
# print(RE_ROW_LIST)
FN = infile.replace(".txt", "S.txt")
Of = open(FN, "w+")
for i in range(len(RE_ROW_LIST)):
for j in range(len(RE_ROW_LIST[i])):
Of.write("%s" % RE_ROW_LIST[i][j])
if (j != len(RE_ROW_LIST[i]) - 1):
Of.write(" ")
if (j == len(RE_ROW_LIST[i]) - 1):
Of.write("\n")
Of.close()
# print(FN)
return FN
]
list_days = []
for i in range(1, 32):
if i / 10 < 1:
list_days.append("0" + str(i))
else:
list_days.append(str(i))
#print(list_days)
for month in list_mon:
for days in list_days:
filename = "C:/Users/manggny/Desktop/data/MagIch_2014/MagIch_2014" + month + days + ".txt"
column = []
try:
column = MakeList_column(filename)
#print('2')
except:
#print('1')
continue
new_c = []
for line in column:
new = []
for i in range(len(line)):
new.append(line[i].replace(",", ''))
new_c.append(new)
val = []
for i in new_c[6]:
if i == 'NaN':
def Make_simulation(self, infilename, EventNum=100):
Filelist = read_file_name(infilename)
pick_all = input("Do you want to Make All MC? (Y/N)\n")
pick_all = pick_all.lower()
print(" * Input file name :", Filelist[2])
if (pick_all != "y"):
Make_MC = PICKING_BRANCH_n_SAVING(Filelist[2])
print(" * Tempary produce file :", Make_MC[0])
print(" * Going to make MC for", Make_MC[1])
INPUT = Make_MC[0]
else:
INPUT = Filelist[2]
data_col_list = MakeList_column(INPUT)
sorted_col_list = []
DATA_LIST = []
DATA_NAME_LIST = []
for i in range(len(data_col_list)):
temp_list = []
for j in range(len(data_col_list[i])):
if (j == 0):
DATA_NAME_LIST.append(data_col_list[i][j])
continue
temp_t = float(data_col_list[i][j])
temp_list.append(temp_t)
temp_list.sort()
sorted_col_list.append(temp_list)
# print(sorted_col_list)
DATA_LIST.append(DATA_NAME_LIST)
for i in range(EventNum):
if (i % 1000 == 0):
print("Now looping", i, "th event")
temp_list1 = []
for j in range(len(sorted_col_list)):
px = random.random()
Num = len(sorted_col_list[j])
float_Num = float(Num)
prob = 1.0 / float_Num
th_num = 0
th_flag = 0
while (th_flag == 0):
if (px > (th_num * prob)):
# if(0 > (th_num * prob)):
# if(1 > (th_num * prob)):
th_num = th_num + 1
else:
th_flag = 1
#th_num = th_num +1
if (th_num >= len(sorted_col_list[j])):
th_num = len(sorted_col_list[j]) - 1
th_num = th_num
Estimate = (px - float(th_num - 1) / float(Num)) * float(
Num) * (sorted_col_list[j][th_num] - sorted_col_list[j][
th_num - 1]) + sorted_col_list[j][th_num - 1]
temp_list1.append(Estimate)
# print(th_num, sorted_col_list[j][th_num-1],Estimate, sorted_col_list[j][th_num] )
DATA_LIST.append(temp_list1)
# print(DATA_LIST)
Write_MC_File = Filelist[3] + Filelist[0] + "_MC.txt"
self.Write_list = DATA_LIST
self.Write_file = Write_MC_File
#print(self.Write_file)
if (pick_all != "y"):
remove_command = "rm -rf " + INPUT
os.system(remove_command)
print(" * Removing Tempary file", INPUT)
self.MakeMCFile_UseRowList()
import sys, os
sys.path.append(
"/Users/leejunho/Desktop/git/python3Env/group_study/project_pre/func")
from d0_makelist_column import MakeList_column
infile = "../SEOUL_Day.txt"
#infile = "/Users/leejunho/Desktop/git/python3Env/group_study/project_pre/data_txt/BEIJING_Aqi/carbon_copied_data/n0_basic_day/Aqi_Beijing_Holi_re.txt"
COL_List = MakeList_column(infile)
Of = open("NEW.txt", "w+")
for i in range(len(COL_List[0])):
if i == 0:
Of.write("%s\n" % COL_List[0][i])
# Of.write("%s TEMP_RANGE\n" %COL_List[0][i])
else:
Of.write("%s\n" % COL_List[0][i])
# Of.write("%s %s\n" %(COL_List[0][i],(str(float(COL_List[12][i])-float(COL_List[10][i])))))
# Of.write("%s %s\n" %(COL_List[0][i],(str(((float(COL_List[10][i])+float(COL_List[12][i]))/2.0)))))
Of.close()
import sys
import os
import matplotlib.pyplot as plt
import pylab as plt
import numpy as np
import matplotlib.legend as legend
import math
sys.path.append("../func")
from d0_makelist_column import MakeList_column
filename = ("../data_txt/ALL_DATA/Aqi_Beijing_monthly.txt")
graph_list = MakeList_column(filename)
graph_list_subject = []
graph_list_body = []
for i in range(len(graph_list)):
Temp_list_subject = []
Temp_list_body = []
for k in range(len(graph_list[i])):
if k == 0:
Temp_list_subject.append(graph_list[i][k])
else:
Temp_list_body.append(float(graph_list[i][k]))
graph_list_subject.append(Temp_list_subject)
graph_list_body.append(Temp_list_body)
print(len(graph_list_body[0]))
month = range(0, 51)
print(month)
A_list = graph_list_body[3]
def PlotProper_YearDATE(filename,title="Development",Xlabel="Date",Ylabel="Object", MSE=1.0):
filelist = read_file_name(filename)
Col_list = MakeList_column(filelist[2])
#print(len(Col_list))
Y_data = []
nomi_X_date = []
List_x = [];
temp_x = []
error_low = []; error_high = [];
error_2low = []; error_2high = [];
sqrt_MSE = math.sqrt(MSE)
MIN_Y=0; MAX_Y=0
for i in range(len(Col_list[1])):
if(i==0):
continue
if(i > 1):
if float(Col_list[1][i])>MAX_Y:
MAX_Y = float(Col_list[1][i])
if float(Col_list[1][i])<MIN_Y:
MIN_Y = float(Col_list[1][i])
Y_data.append(float(Col_list[1][i]))
error_low.append(float(Col_list[1][i])-sqrt_MSE); error_2low.append(float(Col_list[1][i])-2*sqrt_MSE)
error_high.append(float(Col_list[1][i])+sqrt_MSE);error_2high.append(float(Col_list[1][i])+2*sqrt_MSE)
nomi_X_date.append(float(Col_list[0][i]))
List_x.append(str(int(Col_list[0][i])))
temp_x.append(i-1)
if(i==1):
MIN_Y = float(Col_list[1][i])
MAX_Y = float(Col_list[1][i])
RANGE_Y = (MAX_Y - MIN_Y)/5
plt.grid(True)
plt.plot(Y_data,'b^:')
#plt.fill_between(temp_x,error_2low,error_2high,edgecolor='#FF9848', facecolor='#FF9848')
plt.fill_between(temp_x,error_low,error_high,edgecolor='#7EFF99', facecolor='#7EFF99')
#plt.fill_between(temp_x,error_2low,error_2high,edgecolor='#CC4F1B', facecolor='#FF9848')
plt.plot([18,19,20,21],[23.69,23.08,26.17,24.72],'r^-') #Manually input real 2014,2015,2016,2017 DATA
#plt.plot(temp_x,'r:')
plt.title(title)
plt.xlabel(Xlabel,fontsize=16,color='r'); plt.ylabel(Ylabel,fontsize=16,color='r')
plt.axis([-1,len(Col_list[1])-1,0,MAX_Y+RANGE_Y])
#plt.xticks(np.arange(len(Col_list[1])),('Tom', '', '', '', 'Sue'))
if(len(Col_list[1]) < 10):
plt.xticks(np.arange(len(Col_list[1]),nomi_X_date))
else:
XTicks = []
tot_len = len(nomi_X_date)
for i in range(tot_len):
if(i==0):
XTicks.append(List_x[i])
elif(i == int(tot_len/4)):
XTicks.append(List_x[i])
elif(i == int(tot_len/2)):
XTicks.append(List_x[i])
elif(i == int(3*tot_len/4)):
XTicks.append(List_x[i])
elif(i == int(tot_len)-1):
XTicks.append(List_x[i])
break
else:
XTicks.append("")
plt.xticks(np.arange(len(Col_list[1])),XTicks)
#plt.legend(["Predicted PM2.5","Recorded PM2.5",r"$2 \sigma$ Error band", r"$1 \sigma$ Error band"])
#plt.legend(["Predicted PM2.5","Recorded PM2.5",r"$1 \sigma$ Error band"])
#plt.legend(["PM2.5 Estimated","PM2.5 Recorded",r"$\sqrt{MSE}$ Estimated"])
plt.legend(["PM2.5 Estimated","PM2.5 Recorded","Error Estimated"],prop={'size': 12})
plt.show()
def formation(
filename1,
filename2): # it's the function which make file2 same format of file1
if (filename1[0] == "/"):
filename1 = filename1
elif ((filename1[0] == "C") & (filename1[1] == ":")):
filename1 = filename1
else:
filename1 = os.getcwd(
) + "/" + filename1 # get the path included filename
loca1 = len(filename1)
for i in range(1, len(filename1) + 1): # find the "/" location
if (filename1[-i] == "/"):
loca1 = i - 1
break
FILENAME1 = filename1.replace(filename1[:-loca1],
"") # this is the shorten filename
filename1_No_Txt = FILENAME1.replace(".txt", "")
infile1 = filename1
if (filename2[0] == "/"):
filename2 = filename2
elif ((filename2[0] == "C") & (filename2[1] == ":")):
filename2 = filename2
else:
filename2 = os.getcwd(
) + "/" + filename2 # get the path included filename
loca2 = len(filename2)
for i in range(1, len(filename2) + 1): # find the "/" location
if (filename2[-i] == "/"):
loca2 = i - 1
break
FILENAME2 = filename2.replace(filename2[:-loca2],
"") # this is the shorten filename
filename2_No_Txt = FILENAME2.replace(".txt", "")
infile2 = filename2
DATE_ROW = MakeList_column(infile1)
DATA_ROW = MakeList_column(infile2)
form_date = DATE_ROW[0]
data_date = DATA_ROW[0]
list_amount = DATA_ROW[8]
list_volume = DATA_ROW[7]
ch_amount = []
ch_volume = []
j = 1
nones = 0
for i in range(len(form_date)):
if i == 0:
ch_amount.append(list_amount[i])
ch_volume.append(list_volume[i])
continue
else:
while 1:
if j >= (len(data_date) - 1):
ch_amount.append(None)
ch_volume.append(None)
nones += 1
j = 1
break
elif form_date[i] == data_date[j]:
a = list_amount[j].replace(',', '')
b = list_volume[j].replace(',', '')
ch_amount.append(a)
ch_volume.append(b)
break
else:
j += 1
NEWNAME = infile1.replace(".txt", "_Changed_" + filename2_No_Txt)
FN = NEWNAME + ".txt"
Of = open(FN, "w+")
for j in range(len(ch_amount)):
Of.write(str(ch_amount[j]) + " " + str(ch_volume[j]) + "\n")
Of.close()
return FN
def ROW_COL(filename1, filename2):
if (filename1[0] == "/"):
filename1 = filename1
elif ((filename1[0] == "C") & (filename1[1] == ":")):
filename1 = filename1
else:
filename1 = os.getcwd(
) + "/" + filename1 # get the path included filename
loca1 = len(filename1)
for i in range(1, len(filename1) + 1): # find the "/" location
if (filename1[-i] == "/"):
loca1 = i - 1
break
FILENAME1 = filename1.replace(filename1[:-loca1],
"") # this is the shorten filename
filename1_No_Txt = FILENAME1.replace(".txt", "")
infile1 = filename1
if (filename2[0] == "/"):
filename2 = filename2
elif ((filename2[0] == "C") & (filename2[1] == ":")):
filename2 = filename2
else:
filename2 = os.getcwd(
) + "/" + filename2 # get the path included filename
loca2 = len(filename2)
for i in range(1, len(filename2) + 1): # find the "/" location
if (filename2[-i] == "/"):
loca2 = i - 1
break
FILENAME2 = filename2.replace(filename2[:-loca2],
"") # this is the shorten filename
filename2_No_Txt = FILENAME2.replace(".txt", "")
infile2 = filename2
COL_LIST1 = MakeList_column(infile1)
COL_LIST2 = MakeList_column(infile2)
COL_LIST1.append(COL_LIST2[1])
COL_LIST3 = COL_LIST1
# print(COL_LIST3)
ROW_LIST4 = []
for k in range(len(COL_LIST3[0])):
gg = []
for m in range(len(COL_LIST3)):
gg.append(COL_LIST3[m][k])
ROW_LIST4.append(gg)
# print(ROW_LIST4)
NEWNAME = infile1.replace(".txt", "_" + filename2_No_Txt)
FN = NEWNAME + ".txt"
Of = open(FN, "w+")
for i in range(len(ROW_LIST4)):
for j in range(len(ROW_LIST4[i])):
Of.write("%s" % ROW_LIST4[i][j])
if (j != len(ROW_LIST4[i]) - 1):
Of.write(" ")
if (j == len(ROW_LIST4[i]) - 1):
Of.write("\n")
Of.close()
return FN
def line_graph(filename, y_axis):
filename = filename
Raw_list = MakeList_column(filename)
Graph_list_subject = []
Graph_list_body = []
# 데이터를 이름과 값의 리스트로 분리합니다. 이름: Graph_list_subject, 값: Graph_list_value
for i in range(len(Raw_list)):
Temp_list_subject = []
Temp_list_body = []
Temp_list = []
if i == 0:
for k in range(len(Raw_list[i])):
if k == 0:
Temp_list_subject.append(Raw_list[i][k])
else:
Temp_list.append(Raw_list[i][k])
for j in range(len(Temp_list)):
YYYY = Temp_list[j][0:4]
MM = Temp_list[j][4:6]
DD = Temp_list[j][6:8]
Temp_list_body.append(YYYY + '-' + MM + '-' + DD)
else:
for k in range(len(Raw_list[i])):
if k == 0:
Temp_list_subject.append(Raw_list[i][k])
else:
Temp_list_body.append(float(Raw_list[i][k]))
# 그래프용 리스트 완성
Graph_list_subject.append(Temp_list_subject)
Graph_list_body.append(Temp_list_body)
# x축에 날짜 간격 리스트 만들기
def format_date(x):
format_date_list = []
for i in range(len(x)):
if x[i][5:7] == '01' and x[i][8:10] == '01':
k = x[i]
format_date_list.append(k)
#if x[i][5:7] == '06' and x[i][8:10] == '01':
# k = x[i]
# format_date_list.append(k)
#if x[i][5:7] == '09' and x[i][8:10] == '01':
# k = x[i]
# format_date_list.append(k)
#if x[i][5:7] == '12' and x[i][8:10] == '01':
# k = x[i]
# format_date_list.append(k)
else:
k = ""
format_date_list.append(k)
return format_date_list
N = len(Graph_list_body[0])
ind = np.arange(N)
x_label = Graph_list_subject[0][0]
y_label = Graph_list_subject[y_axis][0]
plt.plot(ind, Graph_list_body[y_axis], '-')
plt.xticks(ind, format_date((Graph_list_body[0])))
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(y_label)
plt.show()
def PICKING_BRANCH_n_SAVING(filename1):
if (filename1[0] == "/"):
filename1 = filename1
elif (filename1[0] == '~'):
filename1 = filename1.replace("~", os.environ['HOME'])
elif ((filename1[0] == "C") & (filename1[1] == ":")):
filename1 = filename1
else:
filename1 = os.getcwd(
) + "/" + filename1 # get the path included filename
loca1 = len(filename1)
for i in range(1, len(filename1) + 1): # find the "/" location
if (filename1[-i] == "/"):
loca1 = i - 1
break
FILENAME1 = filename1.replace(filename1[:-loca1],
"") # this is the shorten filename
filename1_No_Txt = FILENAME1.replace(".txt", "")
infile1 = filename1
print(infile1)
RawList = MakeList(infile1)
RawList_c = MakeList_column(infile1)
num = 1
print("Which branch do you want to extract? (Press 'q' to stop slecting)")
for j in RawList[0]:
print(num, ":", j, "", end="\n")
num = num + 1
#print("Please input branch name you want to extract")
List_c = []
print("Please input Branch Name(not branch number)!")
for i in range(len(RawList[0])):
print("Branch Name (Press 'q' to stop slecting): ", end='')
Branch_name = input()
if Branch_name in RawList[0]:
index_num = 0
for j in RawList[0]:
if j == Branch_name:
List_c.append(RawList_c[index_num])
#print(List_c)
break
else:
index_num = index_num + 1
#print(index_num)
elif Branch_name not in RawList[0] and Branch_name != "q":
print("There are no such branches!")
elif Branch_name == "q":
break
FN = infile1.replace(".txt", "_P.txt")
Of = open(FN, "w+")
for i in range(len(RawList)):
for j in range(len(List_c)):
Of.write("%s" % List_c[j][i])
if (j != len(List_c) - 1):
Of.write(" ")
if (j == len(List_c) - 1):
Of.write("\n")
Of.close()
return FN
def man_py_scatter(filename):
# sys.path.append("../../func")
from d0_makelist_column import MakeList_column
path = '../data_txt/BEIJING_Aqi/'
if (filename[0] == "/"):
filename = filename
elif ((filename[0] == "C") & (filename[1] == ":")):
filename = filename
else:
filename = os.getcwd(
) + "/" + filename # get the path included filename
loca = len(filename)
for i in range(1, len(filename) + 1): # find the "/" location
if (filename[-i] == "/"):
loca = i - 1
break
FILENAME = filename.replace(filename[:-loca],
"") # this is the shorten filename
filename_No_Txt = FILENAME.replace(".txt", "")
infile = filename
list_c = MakeList_column(infile)
#date_list = list_c.pop(0)
#print(len(list_c))
x = len(list_c)
size = 150 / len(list_c[0])
#print(len(list_c[0]),len(list_c[1]),len(list_c[2]))
for i in range(x):
y_list_int = []
y_list = []
y_list = list_c[i]
# print(list_c)
# print(y_list)
# print(i)
#print(y_list)
y_label = y_list[0]
x_len = len(list_c)
for k in range(1, len(y_list)):
# print(k)
y_list_int.append(float(y_list[k]))
#print(len(y_list_int))
plt.figure(10)
#print(x_len)
file_num = math.ceil(x_len / 6)
for l in range(file_num):
posi = 0
if (l + 1) * 6 >= x_len:
max_num = x_len
else:
max_num = (l + 1) * 6
for j in range(l * 6, max_num):
posi += 1
x_list_int = []
x_list = []
x_list = list_c[j]
x_label = x_list[0]
for k in range(1, len(x_list)):
x_list_int.append(float(x_list[k]))
#num = row_n*100+row_n*10+1+j
#print(num)
r, p = stats.pearsonr(x_list_int, y_list_int)
r = float("%0.2f" % (r))
p = float("%0.3f" % (p))
a = plt.subplot(3, 2, posi)
plt.scatter(x_list_int,
y_list_int,
s=size,
c='black',
marker='o',
alpha=0.5,
label='C1')
#plt.xlabel(x_label)
plt.text(0.5,
0.9,
x_label,
ha='center',
va='center',
transform=a.transAxes,
fontsize=8)
plt.text(0.8,
0.2,
'r=' + str(r),
ha='left',
va='center',
color='red',
transform=a.transAxes,
fontsize=8)
plt.text(0.8,
0.1,
'p=' + str(p),
ha='left',
va='center',
color='red',
transform=a.transAxes,
fontsize=8)
#plt.ylabel(y_label)
#plt.title('y = '+y_label+'_'+'x_others')
plt.savefig(y_label + '_' + 'others' + '(' + str(l + 1) + ')' +
'.pdf')
plt.close('all')