def test_significance(length): delta_array = [] for i in range(0, 1000): cause, effect = generate_continue_data(200, random.randint( 1, 3)) # random.randint(1,5) cause = normalize(cause) cause = zero_change(cause) effect = normalize(effect) effect = zero_change(effect) cause2effect = calculate_difference3(cause, effect, length) effect2cause = calculate_difference3(effect, cause, length) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause) p = math.pow(2, -(cause2effect - effect2cause)) #p_array.append(p) delta_array.append(cause2effect - effect2cause) fig = plt.figure() axes = fig.add_subplot(111) delta_array.sort(reverse=True) for i in range(len(delta_array)): if delta_array[i] < -math.log(0.05, 2): # 第i行数据,及returnMat[i:,0]及矩阵的切片意思是:i:i+1代表第i行数据,0代表第1列数据 axes.scatter(i, delta_array[i], color='grey') else: axes.scatter(i, delta_array[i], color='green') plt.xlabel('X') plt.ylabel('Y') plt.show()
def test_solar(): temp, solar = read_solar() cause = map(float, solar) effect = map(float, temp) cause = zero_change(cause) effect = zero_change(effect) cause2effect = calculate_difference3(cause, effect, 5) effect2cause = calculate_difference3(effect, cause, 5) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause)
def temperature_test(): indoor_temp, outdoor_temp = read_temperature() cause = outdoor_temp effect = indoor_temp cause = zero_change(cause) effect = zero_change(effect) cause2effect = calculate_difference3(cause, effect, 10) effect2cause = calculate_difference3(effect, cause, 10) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause)
def test_snow(): temp, snow = read_snow() cause = map(float, temp) effect = map(float, snow) cause = normalize(cause) cause = zero_change(cause) effect = normalize(effect) effect = zero_change(effect) cause2effect = calculate_difference3(cause, effect, 10) effect2cause = calculate_difference3(effect, cause, 10) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause)
def stock_test(): x, y = read_stock_price() cause = map(float, x) effect = map(float, y) #cause = zero_change(cause) #effect = zero_change(effect) for i in range(5, 6): cause2effect = calculate_difference3(cause, effect, i) effect2cause = calculate_difference3(effect, cause, i) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause) p = math.pow(2, -(cause2effect - effect2cause)) print p
def ozone_test(): ozone, temp = read_ozone() cause = temp effect = ozone cause = zero_change(cause) effect = zero_change(effect) for i in range(4, 10): cause2effect = calculate_difference3(cause, effect, i) effect2cause = calculate_difference3(effect, cause, i) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause) p = math.pow(2, -(cause2effect - effect2cause)) print p
def test_sin2(c, e): cause = c effect = e #cause = normalize(cause) cause = zero_change(cause) #effect = normalize(effect) effect = zero_change(effect) cause2effect = calculate_difference3(cause, effect, 100) effect2cause = calculate_difference3(effect, cause, 100) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause) p = math.pow(2, -(cause2effect - effect2cause)) print p
def causality_test(): x, y = read_stock_price() a, b = read_ozone() cause = y[0:300] effect = b[0:300] cause = normalize(cause) cause = zero_change(cause) effect = normalize(effect) effect = zero_change(effect) cause2effect = calculate_difference3(cause, effect, 10) effect2cause = calculate_difference3(effect, cause, 10) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause) p = math.pow(2, -(cause2effect - effect2cause)) print p
def test_h(): counter = 0 for i in range(1, 2): #time, control, valve_position, system_pressure, a_pressure, b_pressure, t_pressure, flow, valve_pressure, load = read_hydraulic_pressure('Sheet'+str(i)) #cause = control #effect = flow cause, effect = read_long_data(200) #cause = normalize(cause) #cause = zero_change(cause) #effect = normalize(effect) #effect = zero_change(effect) cause2effect = calculate_difference3(cause, effect, 10) effect2cause = calculate_difference3(effect, cause, 10) print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause) #p = math.pow(2, -(cause2effect - effect2cause)) #if p<0.05: # counter+=1 #print p plt.plot(cause) plt.plot(effect) plt.show()
def test_data(length, array_length): #txtName = "causal_continue_noise_0.4_normal_sample_1000_length_200.txt" #f = file(txtName, "a+") counter11 = 0 counter10 = 0 counter01 = 0 counter00 = 0 counter11_01 = 0 counter10_01 = 0 counter01_01 = 0 counter00_01 = 0 counter_undecided = 0 counter_true = 0 counter_false = 0 counter_undecided2 = 0 counter_true2 = 0 counter_false2 = 0 counter_error_1 = 0 counter_error_2 = 0 p_array_granger1 = [] p_array_granger2 = [] p_array_CUTE1 = [] p_array_CUTE2 = [] p_array_improve_CUTE1 = [] p_array_improve_CUTE2 = [] p_array1 = [] p_array2 = [] p_array_granger = [] for i in range(0, 1000): write_str = "" p = random.randint(1, 3) #effect, test1 = generate_continue_data(200, p) #cause, effect = generate_continue_data(150, p) #cause_tmp = list(cause) #effect_tmp = list(effect) #cause = zero_change(cause) #effect = zero_change(effect) #cause,effect = generate_continue_data_with_change_lag(350,10) cause = GMM(3, array_length) effect = GMM(5, array_length) cause_tmp = list(cause) effect_tmp = list(effect) #effect = forward_shift_continue_data(cause,p) #noise = np.random.normal(0, 0.1, 200) #for j in range(0, 200): # effect[j] = effect[j] + noise[j] #for i in range(0,len(cause)): #cause[i]=math.tanh(cause[i]) #cause[i] = math.pow(math.e,cause[i]) #effect[i] = math.pow(math.e,effect[i]) #cause[i] = math.pow(cause[i],3)/10 #effect[i] = math.pow(effect[i],3)/10 #effect[i]=math.tanh(effect[i]) #effect[i] = math.pow(effect[i],3) #effect = GMM(8,200) #plt.plot(cause) #plt.plot(effect) #plt.show() #cause = normalize(cause) #effect = normalize(effect) #cause = normalize_data(cause) #effect = normalize_data(effect) #cause = zero_change(cause) #effect = zero_change(effect) from scipy.special import expit #for i in range(0,len(effect)): #effect[i]=expit(effect[i]) #effect[i] = 1.0/effect[i] for ii in range(0, len(cause)): write_str = write_str + " " + str(cause[ii]) for jj in range(0, len(effect)): write_str = write_str + " " + str(effect[jj]) #print "cause:" + str(cause) #print "effect:" + str(effect) # effect, test2 = ge_normal_data(p,200) print "Continuous data, Granger causality test" print "cause->effect" p_value_cause_to_effect1 = [] flag1 = False #ce1 = grangercausalitytests([[effect[i], cause[i]] for i in range(0, len(cause))], p) ce_p = granger(cause, effect, -1) #for key in ce1: # p_value_cause_to_effect1.append(ce1[key][0]["params_ftest"][1]) # if ce1[key][0]["params_ftest"][1] < 0.05: # flag1 = True if ce_p < 0.05: flag1 = True print "effect->cause" p_value_effect_to_cause2 = [] flag2 = False #ce2 = grangercausalitytests([[cause[i], effect[i]] for i in range(0, len(cause))], p) ce2_p = granger(effect, cause, -1) #for key in ce2: # p_value_effect_to_cause2.append(ce2[key][0]["params_ftest"][1]) # if ce2[key][0]["params_ftest"][1] < 0.05: # flag2 = True if ce2_p < 0.05: flag2 = True if ce_p < 0.05: p_array_granger1.append(ce_p) elif ce2_p < 0.05: p_array_granger2.append(ce2_p) if flag1 and flag2: print "Continuous data,Granger two-way cause and effect" write_str = write_str + " " + "连续数据,格兰杰双向因果" counter11 += 1 elif flag1 and not flag2: print "Continuous data,Granger correct cause and effect" write_str = write_str + " " + "连续数据,格兰杰正确因果" counter10 += 1 p_array_granger.append(ce_p) elif not flag1 and flag2: print "Continuous data,Granger wrong cause and effect" write_str = write_str + " " + "连续数据,格兰杰错误因果" counter01 += 1 elif not flag1 and not flag2: print "Continuous data,Granger no cause and effect" write_str = write_str + " " + "连续数据,格兰杰没有因果" #break counter00 += 1 #write_str = write_str + " " + str(min(p_value_cause_to_effect1)) + " " + str(min(p_value_effect_to_cause2)) cause2 = get_type_array(cause, length) effect2 = get_type_array(effect, length) print "01 data, Granger causality test" print "cause->effect" p_value_cause_to_effect3 = [] flag3 = False #ce3 = grangercausalitytests([[effect2[i], cause2[i]] for i in range(0, len(cause2))], p) ce3_p = granger(cause2, effect2, -1) #for key in ce3: # p_value_cause_to_effect3.append(ce3[key][0]["params_ftest"][1]) # if ce3[key][0]["params_ftest"][1] < 0.05: # flag3 = True if ce3_p < 0.05: flag3 = True print "effect->cause" p_value_effect_to_cause4 = [] flag4 = False #ce4 = grangercausalitytests([[cause2[i], effect2[i]] for i in range(0, len(cause2))], p) ce4_p = granger(effect2, cause2, -1) #for key in ce4: # p_value_effect_to_cause4.append(ce4[key][0]["params_ftest"][1]) # if ce4[key][0]["params_ftest"][1] < 0.05: # flag4 = True if ce4_p < 0.05: flag4 = True if flag3 and flag4: print "01 data,Granger two-way cause and effect" write_str = write_str + " " + "离散数据,格兰杰双向因果" counter11_01 += 1 elif flag3 and not flag4: print "01 data,Granger correct cause and effect" write_str = write_str + " " + "离散数据,格兰杰正确因果" counter10_01 += 1 elif not flag3 and flag4: print "01 data,Granger wrong cause and effect" write_str = write_str + " " + "离散数据,格兰杰错误因果" counter01_01 += 1 elif not flag3 and not flag4: print "01 data,Granger no cause and effect" write_str = write_str + " " + "离散数据,格兰杰没有因果" counter00_01 += 1 #write_str = write_str + " " + str(min(p_value_cause_to_effect3)) + " " + str(min(p_value_effect_to_cause4)) print delta_ce = calculate_difference3(cause, effect, length) delta_ec = calculate_difference3(effect, cause, length) print 'cause' + ' -> ' + 'effect' + ':' + str(delta_ce) print 'effect' + ' -> ' + 'cause' + ':' + str(delta_ec) if delta_ce > delta_ec and delta_ce - delta_ec >= -math.log(0.05, 2): print "CUTE,correct cause and effect" write_str = write_str + " " + "CUTE,正确因果" counter_true += 1 elif delta_ec > delta_ce and delta_ec - delta_ce >= -math.log(0.05, 2): print "CUTE,wrong cause and effect" write_str = write_str + " " + "CUTE,错误因果" counter_false += 1 else: print "CUTE,undecided" write_str = write_str + " " + "CUTE,未决定" counter_undecided += 1 write_str = write_str + " " + str(pow(2, -abs(delta_ce - delta_ec))) p = math.pow(2, -(delta_ce - delta_ec)) if p < 1: p_array1.append(p) else: p_array2.append(math.pow(2, -(delta_ec - delta_ce))) #f.write(write_str) #f.write("\n") cause = change_to_zero_one(cause_tmp) effect = change_to_zero_one(effect_tmp) cause2effect = bernoulli2(effect, length) - cbernoulli2( effect, cause, length) effect2cause = bernoulli2(cause, length) - cbernoulli2( cause, effect, length) # print 'cause' + ' -> ' + 'effect' + ':' + str(cause2effect) # print 'effect' + ' -> ' + 'cause' + ':' + str(effect2cause) p = math.pow(2, -(cause2effect - effect2cause)) if p < 1: p_array_improve_CUTE1.append(p) else: p_array_improve_CUTE2.append( math.pow(2, -(effect2cause - cause2effect))) cause2effect = bernoulli(effect) - cbernoulli(effect, cause) effect2cause = bernoulli(cause) - cbernoulli(cause, effect) if p < 1: p_array_CUTE1.append(p) else: p_array_CUTE2.append(math.pow(2, -(effect2cause - cause2effect))) print print "*****************************cut line*****************************" print #f.close() print "连续数据,格兰杰因果关系检验:" print "双向因果:" + str(counter11) print "正确因果:" + str(counter10) print "错误因果:" + str(counter01) print "没有因果" + str(counter00) print "-----------------" print "离散数据,格兰杰因果关系检验:" print "双向因果:" + str(counter11_01) print "正确因果:" + str(counter10_01) print "错误因果:" + str(counter01_01) print "没有因果" + str(counter00_01) print "-----------------" print "discret data,snml causality test:" print "correct cause and effect:" + str(counter_true) print "wrong cause and effect:" + str(counter_false) print "no cause and effect:" + str(counter_undecided) print "-----------------" print "01 data,CUTE causality test:" granger_test = (bh_procedure(p_array_granger1, 0.05) + bh_procedure(p_array_granger2, 0.05)) / 1000.0 ourmodel = (bh_procedure(p_array1, 0.05) + bh_procedure(p_array2, 0.05)) / 1000.0 cute = (bh_procedure(p_array_CUTE1, 0.05) + bh_procedure(p_array_CUTE2, 0.05)) / 1000.0 improve_cute = (bh_procedure(p_array_improve_CUTE1, 0.05) + bh_procedure(p_array_improve_CUTE2, 0.05)) / 1000.0 print granger_test print improve_cute print ourmodel return granger_test, ourmodel, cute, improve_cute