def every_day_check(date):
mongo = MONGODB(db='NASDAQ_CORRELATION')
#f = open('results.csv', mode='a')
#data = pd.read_csv('/Users/leotao/Downloads/correlation_list_1.csv')
#data.columns = ['code1', 'code2', 'correlation', 'start', 'end']
#result = data.loc[data['correlation'] >= 0.98]
result = mongo.read_mongo(collection='2015-05-01/2016-05-01',
query={'_id': 'range: 0.94_0.95'})
result = result[0]
code1 = list(result['code1'])
code2 = list(result['code2'])
for n in range(len(result)):
stock1 = code1[n]
stock2 = code2[n]
print(stock1, stock2)
a = get_yahoo(stock1, start='2016-01-01', end=date)
b = get_yahoo(stock2, start='2016-01-01', end=date)
dataa = data_date_match(a,
b,
'2016-01-01',
date,
parameter='Adj_Close')
coin = find_cointegrated_values(dataa)
if coin <= 0.05:
test = Sta_arb_draw(dataa)
df = test.signal_today()
df.to_csv('results.csv', mode='a', header=False, index=False)
def get_data(list_name,collection):
assert type(list_name) is list
all_data=[]
con = MONGODB(db='NASDAQ_ALL')
for i in list_name:
data = con.read_mongo(collection=collection,query={'_id':i})
if len(data)==0:continue
all_data.append(data[0])
return all_data
def get_data(list_name, collection):
assert type(list_name) is list
all_data = []
con = MONGODB(db='NASDAQ_ALL')
for i in list_name:
data = con.read_mongo(collection=collection, query={'_id': i})
if len(data) == 0: continue
all_data.append(data[0])
return all_data
def draw_pairs(a,b,startdate='2015-05-01',enddate='2016-05-01', data = 'Adj_Close'):
con = MONGODB(db='NASDAQ_ALL')
a = con.read_mongo(collection='until_2016-09-09', query={'_id': a})[0]
b = con.read_mongo(collection='until_2016-09-09', query={'_id': b})[0]
graph = data_date_match(a,b,start=startdate,end=enddate,parameter=data)
graph.plot()
pylab.show()
ratio = sum(graph[list(graph.columns)[0]])/sum(graph[list(graph.columns)[1]])
rratio = 1/ratio-1
rratio = math.ceil(rratio)
ratio = math.ceil(ratio)
ll = graph[list(graph.columns)[0]]*rratio-graph[list(graph.columns)[1]]*ratio
print(ratio,rratio)
ll.plot()
pylab.show()
def draw_pairs(a,
b,
startdate='2015-05-01',
enddate='2016-05-01',
data='Adj_Close'):
con = MONGODB(db='NASDAQ_ALL')
a = con.read_mongo(collection='until_2016-09-09', query={'_id': a})[0]
b = con.read_mongo(collection='until_2016-09-09', query={'_id': b})[0]
graph = data_date_match(a, b, start=startdate, end=enddate, parameter=data)
graph.plot()
pylab.show()
ratio = sum(graph[list(graph.columns)[0]]) / sum(graph[list(
graph.columns)[1]])
rratio = 1 / ratio - 1
rratio = math.ceil(rratio)
ratio = math.ceil(ratio)
ll = graph[list(graph.columns)[0]] * rratio - graph[list(
graph.columns)[1]] * ratio
print(ratio, rratio)
ll.plot()
pylab.show()
def every_day_check(date):
mongo = MONGODB(db='NASDAQ_CORRELATION')
#f = open('results.csv', mode='a')
#data = pd.read_csv('/Users/leotao/Downloads/correlation_list_1.csv')
#data.columns = ['code1', 'code2', 'correlation', 'start', 'end']
#result = data.loc[data['correlation'] >= 0.98]
result = mongo.read_mongo(collection='2015-05-01/2016-05-01',query={'_id':'range: 0.94_0.95'})
result = result[0]
code1 = list(result['code1'])
code2 = list(result['code2'])
for n in range(len(result)):
stock1 = code1[n]
stock2 = code2[n]
print(stock1, stock2)
a = get_yahoo(stock1, start='2016-01-01', end=date)
b = get_yahoo(stock2, start='2016-01-01', end=date)
dataa = data_date_match(a, b, '2016-01-01', date, parameter='Adj_Close')
coin = find_cointegrated_values(dataa)
if coin <= 0.05:
test = Sta_arb_draw(dataa)
df = test.signal_today()
df.to_csv('results.csv',mode='a',header=False,index=False)
def hist_data_import_to_mongodb(self):
mongdb = MONGODB(db='ALL_STOCKS')
for i in self._all_codes[list(self._all_codes).index('600335'):]:
print('process code:' + i)
data = get_his_data(i,start='2005-01-01',end='2008-01-01',if_ma=False)
data2 = get_his_data(i,start='2008-01-01',end='2011-01-01',if_ma=False)
data3 = get_his_data(i, start='2011-01-01', end='2014-01-01', if_ma=False)
data4 = get_his_data(i, start='2014-01-01', end='2017-01-01', if_ma=False)
try:
mongdb.write_to_mongo(data,collection=i,id='2005-01-01/2008-01-01')
except Exception as e:
print(e)
try:
mongdb.write_to_mongo(data2, collection=i, id='2008-01-01/2011-01-01')
except Exception as e:
print(e)
try:
mongdb.write_to_mongo(data3, collection=i, id='2011-01-01/2014-01-01')
except Exception as e:
print(e)
try:
mongdb.write_to_mongo(data4, collection=i, id='2014-01-01/2016-09-06')
except Exception as e:
print(e)
def corrcof_write(df_list, startdate, enddate, ff, numbers='0-300'):
assert type(df_list) is list
length = len(df_list)
combination = list(itertools.combinations(range(length), 2))
con = MONGODB(db='NASDAQ_CORRELATION')
i = 0
for comb in combination:
a = comb[0]
b = comb[1]
data = data_date_match(df_list[a],
df_list[b],
start=startdate,
end=enddate,
parameter='Adj_Close')
#print(data)
data = data.pct_change().dropna()
#print(data)
if len(data) == 0: continue
name = data.columns #
print('Process ' + name[0] + '_' + name[1])
ff.write(
str(name[0]) + ',' + str(name[1]) + ',' +
str(data.corr('pearson')[name[0]][1]) + ',' + data.index[0:1][0] +
',' + data.index[-1:][0] + '\n')
#data['Adj_Open'] = 0
#data['Adj_High'] = 0
#data['Adj_Low'] = 0
#for i in range(len(data)):
# k = data['Adj_Close'][i] / data['Close'][i]
# data.loc[i:i+1,'Adj_Open'] = k*data['Open'][i]
# data.loc[i:i + 1, 'Adj_High'] = k * data['High'][i]
# data.loc[i:i + 1, 'Adj_Low'] = k * data['Low'][i]
data['Symbol'] = stock
return data
if __name__=='__main__':
company_list = pd.read_csv('/Users/leotao/Downloads/companylist.csv')
#company_list = company_list[company_list.loc[company_list['Symbol']=='OFIX'].index[0]:] # 从特定断点开始
# print(get_data(company_list.Symbol[0],start='2016-01-01',end='2016-09-09'))
mongo = MONGODB(db='NASDAQ_ALL') # 打开端口
ii = 0
for code in company_list.Symbol: # 所有NASDAQ股票
print('Process : '+code)
data = get_data(code,start='2016-09-09',end='2016-09-21')
if len(data) == 0: continue
try:
mongo.write_to_mongo(data,collection='2016-09-09_2016-09-20',id=code) # 输入数据
except Exception as e:
f = open('log.txt', mode='a')
f.write(code + '\n')
f.write(str(e) + '\n')
continue
if ii ==300: # 防止反爬虫切断
time.sleep(600)
ii=0
#err=sqrt(sum((xr-xn)**2)/n)
#
#print('Linear regression using polyfit')
#print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, ms error= %.3f' % (a,b,ar,br,err))
#
##matplotlib ploting
#title('Linear Regression Example')
#plot(t,x,'g.--')
#plot(t,xn,'k.')
#plot(t,xr,'r.-')
#legend(['original','plus noise', 'regression'])
#
#show()
#
##Linear regression using stats.linregress
#(a_s,b_s,r,tt,stderr)=stats.linregress(t,xn)
#print('Linear regression using stats.linregress')
#print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, std error= %.3f' % (a,b,a_s,b_s,stderr))
##hs300 = get_his_data('000300',index=True)
##print(hs300)
#
##nsq = get_data('^HSI',start='2012-01-01',end='2016-09-14')
##print(nsq)
##mango = MONGODB(db='STOCK_INDEX')
##mango.write_to_mongo(nsq,collection='HSI',id='2012-01-01/2016-09-14')
test = MONGODB('STOCK_INDEX')
data = test.read_mongo('NSDAQ', query={'_id': 'until/2016-09-14'})
data = data[0]
print(data['Close'])
print(data['Open'])
def get_all(collection):
con = MONGODB(db='NASDAQ_ALL')
data = con.read_mongo(collection=collection)
print('fetch total #df :' + str(len(data)))
return data
def get_all(collection):
con = MONGODB(db='NASDAQ_ALL')
data = con.read_mongo(collection=collection)
print('fetch total #df :' + str(len(data)))
return data
#data['Adj_High'] = 0
#data['Adj_Low'] = 0
#for i in range(len(data)):
# k = data['Adj_Close'][i] / data['Close'][i]
# data.loc[i:i+1,'Adj_Open'] = k*data['Open'][i]
# data.loc[i:i + 1, 'Adj_High'] = k * data['High'][i]
# data.loc[i:i + 1, 'Adj_Low'] = k * data['Low'][i]
data['Symbol'] = stock
return data
if __name__ == '__main__':
company_list = pd.read_csv('/Users/leotao/Downloads/companylist.csv')
#company_list = company_list[company_list.loc[company_list['Symbol']=='OFIX'].index[0]:] # 从特定断点开始
# print(get_data(company_list.Symbol[0],start='2016-01-01',end='2016-09-09'))
mongo = MONGODB(db='NASDAQ_ALL') # 打开端口
ii = 0
for code in company_list.Symbol: # 所有NASDAQ股票
print('Process : ' + code)
data = get_data(code, start='2016-09-09', end='2016-09-21')
if len(data) == 0: continue
try:
mongo.write_to_mongo(data,
collection='2016-09-09_2016-09-20',
id=code) # 输入数据
except Exception as e:
f = open('log.txt', mode='a')
f.write(code + '\n')
f.write(str(e) + '\n')
continue
if ii == 300: # 防止反爬虫切断
#err=sqrt(sum((xr-xn)**2)/n)
#
#print('Linear regression using polyfit')
#print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, ms error= %.3f' % (a,b,ar,br,err))
#
##matplotlib ploting
#title('Linear Regression Example')
#plot(t,x,'g.--')
#plot(t,xn,'k.')
#plot(t,xr,'r.-')
#legend(['original','plus noise', 'regression'])
#
#show()
#
##Linear regression using stats.linregress
#(a_s,b_s,r,tt,stderr)=stats.linregress(t,xn)
#print('Linear regression using stats.linregress')
#print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, std error= %.3f' % (a,b,a_s,b_s,stderr))
##hs300 = get_his_data('000300',index=True)
##print(hs300)
#
##nsq = get_data('^HSI',start='2012-01-01',end='2016-09-14')
##print(nsq)
##mango = MONGODB(db='STOCK_INDEX')
##mango.write_to_mongo(nsq,collection='HSI',id='2012-01-01/2016-09-14')
test = MONGODB('STOCK_INDEX')
data = test.read_mongo('NSDAQ',query={'_id':'until/2016-09-14'})
data = data[0]
print(data['Close'])
print(data['Open'])