def insert_data(parameter):
db = MySQLUtils(user='******', passwd='1988', dbname='stock', source='stock')
# print parameter
for stock in parameter:
data = mlab.csv2rec(stock, delimiter='\t')
pattern = re.compile("[SH]*[SZ]*[0-9]{6}", re.IGNORECASE)
m = pattern.findall(stock)
if len(m) == 1:
# print data
db.insert_ohlc_data(data, m[0])
def insert_data(parameter):
db = MySQLUtils(user='******', passwd='1988', dbname='stock', source='stock')
# print parameter
for stock in parameter:
data = mlab.csv2rec(stock, delimiter='\t')
pattern = re.compile("[SH]*[SZ]*[0-9]{6}", re.IGNORECASE)
m = pattern.findall(stock)
if len(m) == 1:
# print data
db.insert_ohlc_data(data, m[0])
def test_something(self):
db = MySQLUtils('root', '1988', 'test', 'stock')
data = db.get_array('sh600741', begin='2015-01-01', end='2016-01-01')
ohlc = OHLCVD(data)
ohlc.add_macd()
ohlc.add_ma(10)
ohlc.add_rsi_feature()
ohlc.add_raise_day(5)
ohlc.add_raise_day(10)
ohlc.add_recent_down_v_turn()
data = ohlc.get_array()
print data.shape[1]
def test_ohlc_add_all_feature(self):
data = pd.read_csv('d:/stock/new-data/SH600741.TXT', sep='\t')
ohlc = OHLCVD(data.values)
ohlc.add_all_ta_feature()
data = ohlc.get_dataframe()
result = data.replace(np.nan, -1.0).replace(np.inf, -2.0).replace(-np.inf, -3.0)
sql_line = '''insert into stock_with_feature values('sh600741', '%s', %s )'''%(result.values[1, 0],
','.join([str(x) for x in result.values[1, 1:]]), )
print sql_line
db = MySQLUtils('root', '1988', 'stock', 'stock_with_feature')
db.execute_sql(sql_line)
self.assertEqual(ohlc.data.shape[1], 149)
def test_ohlc_add_all_feature(self):
data = pd.read_csv('d:/stock/new-data/SH600741.TXT', sep='\t')
ohlc = OHLCVD(data.values)
ohlc.add_all_ta_feature()
data = ohlc.get_dataframe()
result = data.replace(np.nan,
-1.0).replace(np.inf,
-2.0).replace(-np.inf, -3.0)
sql_line = '''insert into stock_with_feature values('sh600741', '%s', %s )''' % (
result.values[1, 0],
','.join([str(x) for x in result.values[1, 1:]]),
)
print sql_line
db = MySQLUtils('root', '1988', 'stock', 'stock_with_feature')
db.execute_sql(sql_line)
self.assertEqual(ohlc.data.shape[1], 149)
ohlc.add_raise_day(5)
ohlc.add_raise_day(3)
ohlc.add_ma(60)
# ohlc.add_rsi_feature()
ohlc.add_jump_empty_down()
ohlc.normalize()
results.append(ohlc.data)
label = ohlc.future_return(5)
labels.append(label)
print len(results)
print len(labels)
return results
if __name__ == '__main__':
db = MySQLUtils('root', '1988', 'stock', 'stock_with_feature')
import pandas as pd
data = pd.read_csv('d:/stock/new-data/sh600741.txt', sep='\t')
result = OHLCVD(data.values)
result.add_all_ta_feature()
result = result.data_frame.replace(np.nan, -1.0).replace(np.inf, -2.0).replace(-np.inf, -3.0)
db_name = "stock_with_feature"
db.create_feature_db(db_name)
db.insert_feature_data(db_name, result.values, 'sh600741')
# result = db.get_array('sh600741', begin='2007-10-10', end='2015-12-12')
# ohlc = OHLCVD(data=result)
# ohlc.add_macd()
# ohlc.add_fall_days(5)
# ohlc.add_raise_day(5)
# ohlc.add_raise_day(3)
# ohlc.add_ma(60)
def build_stock_list(data_path):
stock_list = []
for root, _, files in os.walk(data_path):
for f in files:
stock_list.append(os.path.join(root, f))
return stock_list
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-n', type=int, help='thread number')
parser.add_argument('-path', type=str, help='path to the data')
try:
db = MySQLUtils(user='******',
passwd='1988',
dbname='stock',
source='stock')
# db.create_db("test")
args = parser.parse_args()
print args.n
stock_list = build_stock_list(args.path)
print stock_list
length = len(stock_list) / args.n
stock_cache = [
stock_list[i:i + length] for i in range(0, len(stock_list), length)
]
if len(stock_list) % length != 0:
begin = len(stock_list) / length * length
stock_cache.append(stock_list[begin:begin +
len(stock_list) % length])
print stock_cache
# coding=utf-8
__author__ = 'squall'
import numpy as np
from backtest.utils.mysql import MySQLUtils
from data.ohlc import OHLCVD
if __name__ == '__main__':
db = MySQLUtils('root', '1988', 'stock', 'stock')
result = db.get_array('sh600741', begin='2007-10-10', end='2015-12-12')
ohlc = OHLCVD(data=result)
print result
print ohlc.get_dataframe()
ohlc.add_macd()
ohlc.add_ma(60)
print ohlc.get_array().shape
print ohlc.get_dataframe()
vol_values = ohlc.data_frame.volume.values
kernel = np.ones(20)*0.05
result = np.convolve(kernel, vol_values, mode='valid')
# plt.plot(result)
# plt.show()
second_day = values[1:]
raise_rate = (second_day-first_day)/first_day*100
result = pd.DataFrame(data=raise_rate, index=data_frame.date.values[1:])
return result
def window_similarity(data_frame, frame_len=20, frame_move=10):
result = []
value = data_frame.values
for i in range(0, value.shape[0]-frame_len, frame_move):
tmp_value = value[i:i+frame_len, :]
cor_value = correlation(tmp_value[:, 0], tmp_value[:, 1])
result.append(cor_value)
return result
db = MySQLUtils('root', '1988', 'test', 'stock')
stock_lists = db.execute_sql("SELECT DISTINCT ID FROM STOCK")
stock_lists = [x[0] for x in stock_lists]
def iter_over_stocks(stock_lists, db):
final_result = {}
for x in stock_lists:
for y in stock_lists:
data1 = db.select_data(stock_lists[0], begin='2010-01-01', end='2015-12-30')
data_frame = db.get_dataframe('sh600741', begin='2010-01-01', end='2015-12-30')
data_frame = data_frame.set_index(data_frame.date)
data_frame1 = db.get_dataframe('sh601668', begin='2010-01-01', end='2015-12-30')
data_frame1 = data_frame1.set_index(data_frame1.date.values)
# coding=utf-8
__author__ = 'squall'
import talib
import numpy as np
import pandas as pd
from backtest.utils.mysql import MySQLUtils
db = MySQLUtils('root', '1988', 'test', 'stock')
data = db.get_array(id='sh600741', begin='2010-01-01', end='2016-12-31')
print data
print np.exp(data[:, 5].astype(np.float64))
# note that all ndarrays must be the same length!
inputs = {
'open': data[:, 1].astype(np.float64),
'high': data[:, 2].astype(np.float64),
'low': data[:, 3].astype(np.float64),
'close': data[:, 4].astype(np.float64),
'volume': np.exp(data[:, 5].astype(np.float64))
}
macd_result = talib.abstract.MACD(inputs)
macd_result = np.vstack(macd_result)
macd_data_frame = pd.DataFrame(data=macd_result.T, index=data[:, 0])
x = macd_result.T
y = x[:, 1] - x[:, 0]
print y*2
# macd_data_frame.plot()
raise_rate = (second_day - first_day) / first_day * 100
result = pd.DataFrame(data=raise_rate, index=data_frame.date.values[1:])
return result
def window_similarity(data_frame, frame_len=20, frame_move=10):
result = []
value = data_frame.values
for i in range(0, value.shape[0] - frame_len, frame_move):
tmp_value = value[i:i + frame_len, :]
cor_value = correlation(tmp_value[:, 0], tmp_value[:, 1])
result.append(cor_value)
return result
db = MySQLUtils('root', '1988', 'test', 'stock')
stock_lists = db.execute_sql("SELECT DISTINCT ID FROM STOCK")
stock_lists = [x[0] for x in stock_lists]
def iter_over_stocks(stock_lists, db):
final_result = {}
for x in stock_lists:
for y in stock_lists:
data1 = db.select_data(stock_lists[0],
begin='2010-01-01',
end='2015-12-30')
data_frame = db.get_dataframe('sh600741',
begin='2010-01-01',
end='2015-12-30')
data_frame = data_frame.set_index(data_frame.date)
ohlc.add_raise_day(5)
ohlc.add_raise_day(3)
ohlc.add_ma(60)
# ohlc.add_rsi_feature()
ohlc.add_jump_empty_down()
ohlc.normalize()
results.append(ohlc.data)
label = ohlc.future_return(5)
labels.append(label)
print len(results)
print len(labels)
return results
if __name__ == '__main__':
db = MySQLUtils('root', '1988', 'stock', 'stock_with_feature')
import pandas as pd
data = pd.read_csv('d:/stock/new-data/sh600741.txt', sep='\t')
result = OHLCVD(data.values)
result.add_all_ta_feature()
result = result.data_frame.replace(np.nan, -1.0).replace(np.inf,
-2.0).replace(
-np.inf, -3.0)
db_name = "stock_with_feature"
db.create_feature_db(db_name)
db.insert_feature_data(db_name, result.values, 'sh600741')
# result = db.get_array('sh600741', begin='2007-10-10', end='2015-12-12')
# ohlc = OHLCVD(data=result)
# ohlc.add_macd()
# ohlc.add_fall_days(5)
# ohlc.add_raise_day(5)
