def prob_with_evidence(A, w, L, sample, its, max_prod=False):
n = len(A)
domain = Domain(tuple(range(len(w))))
edge_potential = EdgePotential()
node_potential = NodePotential(w)
rvs = list()
factors = list()
for i in range(n):
rv = RV(domain, value=(int(sample[i]) if L[i] == 0 else None))
rvs.append(rv)
factors.append(F(node_potential, (rv, )))
for i in range(n):
for j in range(n):
if i < j and A[i, j] == 1:
factors.append(F(edge_potential, (rvs[i], rvs[j])))
g = Graph(rvs, factors)
bp = BP(g, max_prod=max_prod)
bp.run(iteration=its)
res = np.zeros((n, len(w)))
for i in range(n):
p = bp.prob(rvs[i])
for k in range(len(w)):
res[i, k] = p[k]
return res
def maxprod(A, w, its):
n = len(A)
domain = Domain(tuple(range(len(w))))
edge_potential = EdgePotential()
node_potential = NodePotential(w)
rvs = list()
factors = list()
for i in range(n):
rv = RV(domain, value=None)
rvs.append(rv)
factors.append(
F(node_potential, (rv,))
)
for i in range(n):
for j in range(n):
if i < j and A[i, j] == 1:
factors.append(
F(edge_potential, (rvs[i], rvs[j]))
)
g = Graph(rvs, factors)
bp = BP(g, max_prod=True)
bp.run(iteration=its)
x = list()
for i in range(n):
x.append(bp.map(rvs[i]))
return x
def sumprod(A, w, its):
n = len(A)
g = build_graph(A, w)
bp = BP(g)
bp.run(iteration=its)
return bp.partition()
def train(samples,
W,
path='.',
BP=MyBP,
t0=30000,
v=0.8,
error=10**-8,
**kwargs):
if 'fname' in kwargs:
fpath = path + '/W_of_' + kwargs['fname']
else:
fpath = path + '/W_of_' + str(np.random.rand())
print('开始模拟退火训练,训练数据保存至' + fpath)
t = t0
W = MyBP.train(samples, W)
best_W = W
while MyBP.E(samples, W) > error and t > 3 * error:
# test
print('t:', t, '\nE:', MyBP.E(samples, W))
for i in range(10):
W_temp = BPSA.genete(W, (-4, 4))
dc = float(BP.E(samples, W_temp) - BP.E(samples, W))
pr = min(1, np.exp(-dc / t))
if pr >= np.random.rand():
W = W_temp
W = MyBP.train(samples,
W,
iterations=1000 +
min(3 * 10**4, int(3 / (t * 10))))
# 保存找到的最优解,如果传入tests,则也验证tests
if 'tests' in kwargs:
flag = MyBP.E(kwargs['tests'], W) < MyBP.E(
kwargs['tests'], best_W)
else:
flag = True
if MyBP.E(samples, W) < MyBP.E(samples, best_W) and flag:
best_W = W
# test
y = BP.output_y(samples[-1][0], W)
try:
print(MyBP.inverse_sigmoid(y))
except ValueError as er:
print(er)
t = v * t
W = MyBP.train(samples, W, iterations=3 * 10**4)
s = shelve.open(fpath)
s['0'] = W
s['best'] = best_W
s.close()
return W
def maxprod(A, w, its):
n = len(A)
g = build_graph(A, w)
rvs = g.rvs
bp = BP(g, max_prod=True)
bp.run(iteration=its)
x = list()
for i in range(n):
x.append(bp.map(rvs[i]))
return x
def prob(A, w, its, max_prod=False):
n = len(A)
g = build_graph(A, w)
rvs = g.rvs
bp = BP(g, max_prod=max_prod)
bp.run(iteration=its)
res = np.zeros((n, len(w)))
for i in range(n):
p = bp.prob(rvs[i])
for k, color in enumerate(w):
res[i, k] = p[k]
return res
def __init__(self, *args, **kwds):
MyFrame.__init__(self, *args, **kwds)
self.Bind(wx.EVT_BUTTON, self.onBPNow, self.bpNow_Button)
self.timer1 = wx.Timer(self)
self.pressure_timer = wx.Timer(self)
self.Bind(wx.EVT_TIMER, self.get_bp, self.timer1)
self.Bind(wx.EVT_TIMER, self.pressure_update, self.pressure_timer)
self.bp = BP(self,'COM6')
self.bp_pressure_indicator = wx.Gauge(self.bpbarpanel,-1, 250, size=(50, 100),style=wx.GA_VERTICAL)
self.bp_pressure_indicator.Enable(False)
self.bp_status_check()
class bpdemo(MyFrame):
def __init__(self, *args, **kwds):
MyFrame.__init__(self, *args, **kwds)
self.Bind(wx.EVT_BUTTON, self.onBPNow, self.bpNow_Button)
self.timer1 = wx.Timer(self)
self.pressure_timer = wx.Timer(self)
self.Bind(wx.EVT_TIMER, self.get_bp, self.timer1)
self.Bind(wx.EVT_TIMER, self.pressure_update, self.pressure_timer)
self.bp = BP(self,'COM6')
self.bp_pressure_indicator = wx.Gauge(self.bpbarpanel,-1, 250, size=(50, 100),style=wx.GA_VERTICAL)
self.bp_pressure_indicator.Enable(False)
self.bp_status_check()
def onBPNow(self, event): # wxGlade: MyFrame.<event_handler>
self.get_bp()
def get_bp(self):
self.bp_pressure_indicator.Enable(True)#enable the vertical bar pressurte reading
self.bpNow_Button.Enable(False)#disable the bp acquire button until the bp reaidng is finished
self.bp.send_request()
self.pressure_timer.Start(200)#updates the readings every 200ms
self.count=1
def pressure_update(self,event):
#print "timer for pressure"
press = self.bp.get_reply()
self.bp.nibp.read(1)
#print "pressure: ", press, " mmHg"
press = int(press[1:4])
print self.count
print press
if press != 999:
self.bpNow_Button.Enable(False)
self.bp_pressure_indicator.SetValue(press)
self.bp_infolabel.SetLabel(str(press)+' mmHg')
self.count=0
#self.bp_pressure_indicator.SetValue(press)
else:
self.bp_pressure_indicator.SetValue(0)
self.bp_infolabel.SetLabel('BP Acquired')
self.bp_pressure_indicator.Enable(False)
self.bpNow_Button.Enable(True)
self.bp.get()
self.pressure_timer.Stop()
def bp_status_check(self):
self.bp.POST()
self.bp.device_ready()
self.bp_infolabel.SetLabel(self.bp.device_message)
def updateBPDisplay(self, data):
self.bpvalue_label.SetLabel(data)
import func as f
from BP import BP
if __name__ == "__main__":
x1, y1 = f.generate_linear(n=100)
x2, y2 = f.generate_XOR_easy()
inp = int(input())
if (inp == 1):
network1 = BP(x1, y1)
network1.train()
network1.test()
f.show_result(network1.x, network1.y, network1.pred_y)
else:
network2 = BP(x2, y2)
network2.train()
network2.test()
f.show_result(network2.x, network2.y, network2.pred_y)
def deltaW2(sample, W):
input_p = sample[0]
target = sample[1]
y = MyBP.output_y(input_p, W)
return (y - BP.sigmoid(target)) * y * (1 - y)
from net import Net
import visdom
from dataset import DataSet
from BP import BP
config = {
'lr': 0.0001,
'max_epoch': 500,
'vis': visdom.Visdom(env=u'plus_loss_test'),
'net': Net(2, [5, 3, 1], 0.0001, 'relu'),
'data': DataSet('x1+x2')
}
process = BP(config=config)
process.run()
import sys
sys.setrecursionlimit(1000000)
K = 10
X, Y, X_valid, Y_valid, X_test, Y_test = init_data(K=K)
# X, Y, X_valid, Y_valid, X_test, Y_test = init_data(X_test, Y_test, K=K)
bp = BP(X,
Y,
X_valid=X_valid,
Y_valid=Y_valid,
X_test=X_test,
Y_test=Y_test,
K=10,
iter_num=1000,
hidden_num=2,
hidden_cell_num=100,
alpha=0.3,
labd=0.006,
threshold=0,
is_save_module=False,
is_use_module=False)
# bp.check_gd() # 梯度检验
bp.fit()
# bp.predict()
# bp.predict(X_test, Y_test, text='acc_test')
# bp.learn_curve()
def run(self, n_jobs, date=None):
# 当date有输入时，为目标日期，当没有输入时，为每天执行，定时刷新上一天的数据
if date:
dt_today = dtparser.parse(str(date))
dt_today = self.calendar[self.calendar > dt_today].iloc[0]
else:
dt_today = dtparser.parse(
datetime.datetime.now().strftime('%Y%m%d'))
if self.calendar[self.calendar == dt_today].empty:
print('Not Trade Day...')
else:
dt_last_trade_day = self.calendar[
self.calendar < dt_today].iloc[-1]
dt_last_week = dt_last_trade_day - relativedelta(weeks=1)
dt_last_1yr = dt_last_trade_day - relativedelta(years=1)
last_trade_day = dt_last_trade_day.strftime('%Y%m%d')
last_week = dt_last_week.strftime('%Y%m%d')
last_1yr = dt_last_1yr.strftime('%Y%m%d')
# ---------------------------------------------
# 更新每日行情
self.logger.info(
'//////////////////////////////////////////////////////')
self.logger.info('Date: %s' % (dt_today.strftime('%Y%m%d')))
self.logger.info(
'BeginTime: %s' %
datetime.datetime.now().strftime('%Y%m%d-%H:%M:%S'))
self.logger.info(
'//////////////////////////////////////////////////////\n')
# ---------------------------------------------
self.logger.info(
'******************************************************')
self.logger.info(
'===================基础数据日常更新=====================')
self.logger.info(
'******************************************************')
mkt = mkt_ohlc()
res = mkt.process_data(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------------market data finish------------------')
st = isst()
res = st.process_data(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------st data finish---------------------')
ex = ex_right()
res = ex.process_data(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------ex right finish--------------------')
swap = stk_swap()
res = swap.process_data(last_1yr, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------swap data finish-------------------')
indu = StkIndustry()
res = indu.process_data(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------------stk industry finish-----------------')
idx_weight = index_weight()
res = idx_weight.process_data(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------------index weight finish-----------------')
stk_info = StkInfo()
res = stk_info.process_data(n_jobs)
self.log_res(res)
self.logger.info(
'--------------------stk info finish-------------------')
sh = shares_turnover()
res = sh.process_data(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------shares and turnover finish---------------')
# ---------------------------------------------
self.logger.info(
'******************************************************')
self.logger.info(
'===================财报数据日常更新=====================')
self.logger.info(
'******************************************************')
bs = BalanceSheetUpdate()
res = bs.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'----------------balance sheet finish------------------')
income = IncomeUpdate()
res = income.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------income finish----------------------')
cf = CashFlowUpdate()
res = cf.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------------cash flow finish--------------------')
QnTTM = QnTTMUpdate()
res = QnTTM.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------QnTTM finish---------------------')
wi = WindIndicatorUpdate()
res = wi.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-----------------WindIndicator finish-----------------')
bank = Banks()
res = bank.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------Banks finish----------------------')
loan = LoanClassification()
res = loan.cal_factors(n_jobs)
self.log_res(res)
self.logger.info(
'---------------------Loan finish----------------------')
sec = SecMonthlyReport()
res = sec.cal_factors(20100101, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------------Securities finish-------------------')
insur = InsurIndicators()
res = insur.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------Insurance finish-------------------')
# ---------------------------------------------
self.logger.info(
'******************************************************')
self.logger.info(
'===================因子数据日常更新=====================')
self.logger.info(
'******************************************************')
sz = market_cap()
res = sz.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------------market_cap finish-------------------')
risk_exp = RiskFactorsExposure()
res = risk_exp.cal_factors(last_week, last_trade_day)
self.log_res(res)
self.logger.info(
'----------------risk exposure finish------------------')
risk_cov = RiskCov()
res = risk_cov.cal_factors(last_week, last_trade_day)
self.log_res(res)
self.logger.info(
'------------------risk cov finish---------------------')
spec_risk = SpecificRisk()
res = spec_risk.cal_factors(last_week, last_trade_day)
self.log_res(res)
self.logger.info(
'-----------------spec risk finish---------------------')
ep = EPandEPcut()
res = ep.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------EP finish------------------------')
bp = BP()
res = bp.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------BP finish------------------------')
sp = SP()
res = sp.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------SP finish------------------------')
ncfp = NCFP()
res = ncfp.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------NCFP finish-----------------------')
ocfp = OCFP()
res = ocfp.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------OCFP finish-----------------------')
dp = DP()
res = dp.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------DP_LYR finish---------------------')
ao = Audit()
res = ao.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------Audit finish---------------------')
roe2 = ROE2()
res = roe2.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------ROE2 finish-----------------------')
rnoa = RNOA2()
res = rnoa.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------RNOA finish-----------------------')
roa = ROA2()
res = roa.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------ROA2 finish-----------------------')
cfroi = CFROI2()
res = cfroi.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------CFROI finish----------------------')
gpoa = GPOA2()
res = gpoa.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------GPOA finish-----------------------')
em = EBITMargin()
res = em.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-----------------EBITMargin finish--------------------')
emd = EBITMarginDelta()
res = emd.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------EBITMargin_Delta finish-----------------')
npg = net_profit_growth()
res = npg.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------net profit growth finish---------------')
org = oper_rev_growth()
res = org.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'----------------oper rev growth finish----------------')
tlg = TotLiabGrowth()
res = tlg.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'----------------tot liab growth finish----------------')
surp = Surprise()
res = surp.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------------Surprise finish---------------------')
mr = MarginalROE()
res = mr.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'----------------Marginal ROE finish-------------------')
amihud = Amihud()
res = amihud.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------Amihud finish----------------------')
ln_turnover = LnTurnover()
res = ln_turnover.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-----------------LnTurnover finish--------------------')
maxret = MaxRet()
res = maxret.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------MaxRet finish----------------------')
pr = PeriodRet()
res = pr.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-----------------PeriodRet finish---------------------')
cgo = CGO()
res = cgo.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------CGO finish------------------------')
cov_div = coverage_and_divergence()
res = cov_div.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------anlst cov div finish-------------------')
cnp = consensus_net_profit()
res = cnp.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-----------consensus net profit finish----------------')
epfy1 = EP_FY1()
res = epfy1.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'--------------------EP_FY1 finish---------------------')
score = score_TPER()
res = score.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'----------------score and TPER finish-----------------')
roefy1 = ROE_FY1()
res = roefy1.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------ROE_FY1 finish---------------------')
peg = PEG()
res = peg.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------PEG finish-----------------------')
npfyg = NetProfitFY1_growth()
res = npfyg.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------NetProfitFY1_growth finish---------------')
npll = NPL_leverage()
res = npll.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-----------------NPL leverage finish------------------')
npldiff = NPL_diff()
res = npldiff.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------------NPL diff finish--------------------')
pcg = provision_cov_growth()
res = pcg.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'-------------provision cov growth finish--------------')
iig = InterestIncome_growth()
res = iig.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'------------interest income growth finish-------------')
bpm = BP_M()
res = bpm.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------BP_M finish----------------------')
epm = EP_M()
res = epm.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------EP_M finish----------------------')
spm = SP_M()
res = spm.cal_factors(last_week, last_trade_day, n_jobs)
self.log_res(res)
self.logger.info(
'---------------------SP_M finish----------------------')
self.logger.info(
'//////////////////////////////////////////////////////')
self.logger.info(
'EndTime: %s' %
datetime.datetime.now().strftime('%Y%m%d-%H:%M:%S'))
self.logger.info(
'//////////////////////////////////////////////////////\n')
def addtrain(self, iter, stride, dim):
'''
Add a weak trainer
Arguments:
iter: iterations to train weak BP classifier
stride: learning rate to train weak BP classifier
'''
D = self.data_weight
L = BP()
L.study_rate = stride
L.dim_hide = dim
L.set_weight(D)
L.init_network(self.train_x / 255.0, self.train_y)
for t in range(iter):
L.train(t)
e = L.error()
a = 0.5 * math.log((1 - e) / e)
print(a)
if a < 0: return a
self.subtrainers.append((L))
self.alpha.append(a)
for j in range(self.m):
if L.predict(self.train_x[j]) == self.train_y[j]:
D[j] = D[j] * math.exp(-a)
else:
D[j] = D[j] * math.exp(a)
sumD = np.sum(D)
for j in range(self.m):
D[j] = D[j] / sumD
self.data_weight = D
return a
