opts['randomSeed'] = 2

opts['inputImageSize'] = 224

opts['trainEpoches'] = 100

opts['alldata'] = 572638 # all data 57397

opts['databatch'] = 10

opts['trainWeightDecay'] = 5e-04 # 5e-04

opts['stddev'] = 0.1

opts['database_candle'] = './data/data_candlestick'

opts['database_mat'] = './data/data_mat'

opts['choose_num'] = 2

opts['tradeday'] = '2017-06-01'

opts['path2save'] = './outputs/'

opts['output'] = opts['path2save']+ opts['tradeday'] + '/'

opts['metafile'] = './model/model_epoch.ckpt.meta'

opts['modelname'] = './model/model_epoch.ckpt'

# Create figure with 3x3 sub-plots.

fig, axes = plt.subplots(2, num)

# fig.subplots_adjust(hspace=0.3, wspace=0.3)

for i, ax in enumerate(axes.flat):

if i < num :

# Plot image.

ax.imshow(image1[i,:,:,:])

xlabel = 'Input{}'.format(i)

ax.set_xlabel(xlabel)

ax.set_xticks([])

ax.set_yticks([])

else:

ax.imshow(image2[i-num,:,:,:])

xlabel = 'Reconstruction{}'.format(i-num)

ax.set_xlabel(xlabel)

ax.set_xticks([])

ax.set_yticks([])

# VGG_MEAN = [103.939, 116.779, 123.68]

choose = [datalist[i] for i in batch]

lens = len(batch)

imagebatch = np.zeros([lens, opts['inputImageSize'], opts['inputImageSize'], 3], dtype=np.float32)

for i in range(lens):

img = cv2.imread(choose[i])

img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)

img = img[:, 55:540, :]

img = cv2.resize(img, (opts['inputImageSize'], opts['inputImageSize'])) / 255

imagebatch[i, :, :, :] = img

bgrbatch = imagebatch

tradedays = ts.trade_cal()

tradedays = tradedays[tradedays.isOpen == 1]

tradedays.index = [i for i in range(len(tradedays))]

start_index = tradedays[tradedays.calendarDate == opts['tradeday']].index

tradeday_end = tradedays.iloc[start_index + 19].calendarDate.values.tolist()[0]

tradeday_test_start = tradedays.iloc[start_index + 20].calendarDate.values.tolist()[0]

if os.path.exists(opts['output']+'index_stocks_for_{}.txt'.format(opts['tradeday'])):

with open(opts['output']+'index_stocks_for_{}.txt'.format(opts['tradeday']), 'r') as f:

stocks_lists = f.read().split('\n')

stocks_lists = np.array(stocks_lists)

null_index = np.where(stocks_lists == '')

stocks_lists = np.delete(stocks_lists, null_index)

return stocks_lists.tolist(), []

stocks_lists = []

stocks_mat_lists = []

stocks_filespath = os.listdir(opts['database_candle'])

stocks_matpath = os.listdir(opts['database_mat'])

stocks_mat_collect = []

# # 先取100支股票

# stocks_filespath = stocks_filespath[0:50]

stocks_num = len(stocks_filespath)

start_day = opts['tradeday']

end_day, _ = TradeDay(opts)

for i in range(stocks_num):

# st_name = os.path.join(opts['database_candle'],stocks_filespath[i])

# st_mat_name = os.path.join(opts['database_mat'], stocks_matpath[i])

st_name = opts['database_candle'] + '/' + stocks_filespath[i]

st_mat_name = opts['database_mat'] + '/' + stocks_matpath[i]

st_list = os.listdir(st_name)

st_mat_list = os.listdir(st_mat_name)

st_index = ''

st_choose = ''

for j in range(len(st_list)):

index = st_list[j].find(start_day)

if index == 0 and st_list[j].find(end_day) != -1:

st_index = st_list[j]

st_mat_index = st_mat_list[j]

stocks_mat_collect.append(stocks_matpath[i])

break

if st_index == '':

print(st_name)

continue # 直接舍弃无数据的股票

# assert st_index != ''

st_choose = os.path.join(st_name, st_index)

st_mat_choose = os.path.join(st_mat_name, st_mat_index)

stocks_lists.append(st_choose)

stocks_mat_lists.append(st_mat_choose)

# assert len(stocks_lists) == stocks_num

print(len(stocks_lists))

with open(opts['output']+'index_stocks_for_{}.txt'.format(opts['tradeday']), 'w') as f:

for i in range(len(stocks_lists)):

f.write(stocks_lists[i] + '\n')

with open(opts['output']+'index_stocks_mat_for_{}.txt'.format(opts['tradeday']), 'w') as f:

for i in range(len(stocks_mat_lists)):

f.write(stocks_mat_lists[i] + '\n')

sharpe_ratio_df = pd.DataFrame(columns=['codes', 'sharpes', 'return_20days'])

sharpe_ratio_df['codes'] = stocks_mat_collect

sharpes = []

return_during_20days = []

for i in range(len(stocks_mat_lists)):

data = scio.loadmat(stocks_mat_lists[i])

data = np.array(data['Prices'])

data = (data[:,3] - data[:,0]) / data[:,0]

# sharpe_ratio

data_mean = np.mean(data)

data_std = np.std(data)

sharpe_ratio = data_mean / data_std

sharpes.append(sharpe_ratio)

# 20 day return

data = data.tolist()

data_str = ','.join(str(data[j]) for j in range(len(data)))

return_during_20days.append(data_str)

sharpe_ratio_df['sharpes'] = sharpes

sharpe_ratio_df['return_20days'] = return_during_20days

sharpe_ratio_df.to_excel(opts['output']+'sharpe_ratios_for_{}.xls'.format(opts['tradeday']), sheet_name='sheet1')

datalen = len(datalist)

dataindex = [i for i in range(datalen)]

# batch_num = datalen // opts['databatch']

datafeatures = np.ones(shape=[datalen, 512])

imagecollection = np.zeros(shape=[datalen, 224, 224, 3])

re_imgcollection = np.zeros(shape=[datalen, 224, 224, 3])

# 加载模型

config = tf.ConfigProto()

config.gpu_options.allow_growth = True

sess = tf.Session(config=config)

if opts['metafile']:

saver = tf.train.import_meta_graph(opts['metafile'])

saver.restore(sess, opts['modelname'])

else:

raise Exception('No history model file found!')

graph = tf.get_default_graph()

inputImage = graph.get_tensor_by_name('inputImage:0')

reconstruction = graph.get_tensor_by_name('reconstruction:0')

ave_pooling = graph.get_tensor_by_name('new_encoder_net/ave_pooling:0')

# lossOp = graph.get_tensor_by_name('Loss/lossOp:0')

# global_step = graph.get_tensor_by_name('global_step:0')

# 获取数据

sampleNum = 0

while sampleNum < datalen:

batch = np.min([opts['databatch'], datalen - sampleNum])

batch_index = dataindex[sampleNum : sampleNum + batch]

imagebatch = GetRandBatch(datalist, batch_index, opts)

# imgweight = np.ones(np.shape(imagebatch))

feature, outputs = sess.run([ave_pooling,reconstruction], feed_dict={inputImage: imagebatch})

feature = sess.run(ave_pooling, feed_dict={inputImage: imagebatch})

datafeatures[sampleNum : sampleNum + batch, :] = feature

imagecollection[sampleNum : sampleNum + batch, :,:,:] = imagebatch

re_imgcollection[sampleNum : sampleNum + batch, :, :, :] = outputs

sampleNum += batch

# name_index = time.strftime('%H_%M_%S', time.localtime(time.time()))

plt.figure(1)

plot_images(imagecollection, re_imgcollection, 5)

plt.savefig(opts['output']+'rebuild_for_{}'.format(opts['tradeday']))

plt.close()

# 数据处理

# np.savetxt一般把array数组保存为txt

np.savetxt(opts['output']+'features_for_{}.txt'.format(opts['tradeday']), datafeatures, delimiter=',')

# datafeatures = np.loadtxt(opts['output']+'features_for_{}.txt'.format(opts['tradeday']), delimiter = ',')

stock_num = np.shape(datafeatures)[0]

with open(opts['output']+'index_stocks_for_{}.txt'.format(opts['tradeday']), 'r') as f:

stock_index = f.readlines()

stock_name = []

for i in range(len(stock_index)):

find_code = False

sl = re.split(r'[\/\\]+', stock_index[i])

for j in range(len(sl)):

if re.match(r'\d{6}', sl[j]):

stock_name.append(sl[j])

find_code = True

assert find_code

assert len(stock_name) == stock_num

cos_similarity = cosine(datafeatures)

# np.savetxt('./output/cosine_similarity_for_{}.txt'.format(opts['tradeday']), cos_similarity, delimiter=',')

feature_ungraph = pd.DataFrame(columns=['vertex1', 'vertex2', 'weights'])

vertex1 = []

vertex2 = []

weights = []

for i in list(range(stock_num))[0:-1]:

for j in list(range(stock_num))[i+1:]:

vertex1.append(stock_name[i])

vertex2.append(stock_name[j])

weights.append(cos_similarity[i, j])

feature_ungraph['vertex1'] = vertex1

feature_ungraph['vertex2'] = vertex2

feature_ungraph['weights'] = weights

feature_ungraph.to_excel(opts['output']+'feature_graph_for_{}.xls'.format(opts['tradeday']), sheet_name='sheet1')

# feature_ungraph = pd.read_excel(opts['output']+'feature_graph_for_{}.xls'.format(opts['tradeday']),

# sheet_name='sheet1')

# 网络生成

length = len(feature_graph)

edges = []

edge_weights = []

for i in range(length):

# tuple(节点1, 节点2, 权值)

edges.append(tuple(['{:0>6}'.format(int(feature_graph.loc[i][0])),

'{:0>6}'.format(int(feature_graph.loc[i][1])),

feature_graph.loc[i][2]]))

edge_weights.append(feature_graph.loc[i][2])

graph = IGraph.TupleList(edges=edges, directed=False, weights=True)

modularity_graph = louvain.find_partition(graph, louvain.ModularityVertexPartition, weights=edge_weights)

# vertex_count = graph.vcount()

graph.vs['label'] = graph.vs['name']

mode_num = len(modularity_graph)

modularity_index = np.zeros(len(graph.vs['name']),)

for i in range(mode_num):

modularity_index[modularity_graph[i]] = i

modularity_index = list(map(int, modularity_index))

graph.vs['modularity'] = modularity_index

mode_list = []

for i in range(mode_num):

vertexes = graph.vs.select(modularity=i)

one_mode_list = [vertexes[j]['name'] for j in range(len(vertexes))]

mode_list.append(one_mode_list)

with open(opts['output'] + 'modularity_for_{}.txt'.format(opts['tradeday']), 'w') as f:

for i in range(mode_num):

f.write(','.join(mode_list[i]) + '\n')

# 作图

color_dict = {0:'red', 1:'green', 2:'blue'}

graph.vs['color'] = [color_dict[index] for index in graph.vs['modularity']]

ig.plot(graph)

assert os.path.exists(opts['output'] + 'modularity_for_{}.txt'.format(opts['tradeday']))

with open(opts['output'] + 'modularity_for_{}.txt'.format(opts['tradeday']), 'r') as f:

modularity = f.read()

modularity = modularity.strip('\n').split('\n')

if not os.path.exists(opts['output'] + 'sharpe_ratios_for_{}.xls'.format(opts['tradeday'])):

_, _ = GetStocks(opts)

stocks_sharpe_ratio = pd.read_excel(opts['output'] + 'sharpe_ratios_for_{}.xls'.format(opts['tradeday']),

sheet_name='sheet1')

stocks_dict = {}

for i in range(len(modularity)):

stock_df = pd.DataFrame(columns=['codes', 'sharpes'])

stock_list = modularity[i].split(',')

stock_sharpes = np.array([])

for j in range(len(stock_list)):

sharpe = stocks_sharpe_ratio[stocks_sharpe_ratio.codes == int(stock_list[j])].sharpes.values

stock_sharpes = np.append(stock_sharpes, sharpe)

stock_df['codes'] = stock_list

stock_df['sharpes'] = stock_sharpes.tolist()

stocks_dict[str(i)] = stock_df

choose_dict = {}

for i in range(len(modularity)):

stock_sharpes = stocks_dict[str(i)].sort_values(by= 'sharpes', ascending=False)

choose_dict[str(i)] = list(stock_sharpes[0:opts['choose_num']].codes.values)

assert os.path.exists(opts['output'] + 'modularity_for_{}.txt'.format(opts['tradeday']))

with open(opts['output'] + 'modularity_for_{}.txt'.format(opts['tradeday']), 'r') as f:

modularity = f.read()

modularity = modularity.strip('\n').split('\n')

if not os.path.exists(opts['output'] + 'sharpe_ratios_for_{}.xls'.format(opts['tradeday'])):

_, _ = GetStocks(opts)

stocks_sharpe_ratio = pd.read_excel(opts['output'] + 'sharpe_ratios_for_{}.xls'.format(opts['tradeday']),

sheet_name='sheet1')

modularity_list = []

for i in range(len(modularity)):

stock_list = modularity[i].split(',')

stocks_list = []

for j in range(len(stock_list)):

return_value = stocks_sharpe_ratio[stocks_sharpe_ratio.codes == int(stock_list[j])].return_20days.values

stocks_list.append(return_value)

modularity_list.append(stocks_list)

for i in range(len(modularity_list)):

modularity_stock = modularity_list[i]

for j in range(len(modularity_stock)):

x_data = [k for k in range(21)]

y_data = modularity_stock[j].tolist()[0].split(',')

y_data = [float(y_data[k]) for k in range(len(y_data))]

y_data = np.array([0] + y_data) + 1

y_data = np.cumprod(y_data)

plt.plot(x_data, y_data)

plt.title('20days-return-for-the-{}th-modularity'.format(i))

plt.xlabel('days')

plt.ylabel('cumulative-return')

plt.savefig(opts['output'] + '{}th_return_for_{}'.format(i, opts['tradeday']))

assert os.path.exists(opts['output'] + 'modularity_for_{}.txt'.format(opts['tradeday']))

with open(opts['output'] + 'modularity_for_{}.txt'.format(opts['tradeday']), 'r') as f:

modularity = f.read()

modularity = modularity.strip('\n').split('\n')

print('训练数据开始于：{}'.format(opts['tradeday']))

_, test_start_day = TradeDay(opts)

# test next 10 days

opts['tradeday'] = test_start_day

if not os.path.exists(opts['output'] + 'sharpe_ratios_for_{}.xls'.format(opts['tradeday'])):

_, _ = GetStocks(opts)

stocks_sharpe_ratio = pd.read_excel(opts['output'] + 'sharpe_ratios_for_{}.xls'.format(opts['tradeday']),

sheet_name='sheet1')

print('测试数据开始于：{}'.format(opts['tradeday']))

modularity_list = []

for i in range(len(modularity)):

stock_list = modularity[i].split(',')

stocks_list = []

for j in range(len(stock_list)):

return_value = stocks_sharpe_ratio[stocks_sharpe_ratio.codes == int(stock_list[j])].return_20days.values

if return_value:

stocks_list.append(return_value)

else:

print('no complete data for {}'.format(stock_list[j]))

modularity_list.append(stocks_list)

for i in range(len(modularity_list)):

modularity_stock = modularity_list[i]

for j in range(len(modularity_stock)):

x_data = [k for k in range(11)]

y_data = modularity_stock[j].tolist()[0].split(',')

y_data = [float(y_data[k]) for k in range(len(y_data))]

y_data = y_data[0:10]

y_data = np.array([0] + y_data) + 1

y_data = np.cumprod(y_data)

plt.plot(x_data, y_data)

plt.title('20days-return-for-the-{}th-modularity'.format(i))

plt.xlabel('days')

plt.ylabel('cumulative-return')

plt.savefig(opts['output'] + 'test_{}th_return_for_{}'.format(i, opts['tradeday']))

opts = {}

opts = getOpts(opts)

if not os.path.exists(opts['output']):

os.makedirs(opts['output'])

# t1 = time.time()

datalist, _ = GetStocks(opts)

# print('获取数据用时：{}s'.format(time.time()-t1))

# 卷积自编码器生成特征

# datafeatures = get_feature_from_vgg(datalist, opts)

# datafeatures = np.loadtxt(opts['output'] + 'features_for_{}.txt'.format(opts['tradeday']), delimiter=',')

# feature_graph = get_excel_graph_model(datafeatures, opts)

feature_graph = pd.read_excel(opts['output']+'feature_graph_for_{}.xls'.format(opts['tradeday']), sheet_name='sheet1')

# 模块化聚类分析

graph, modularity =modularity_analysis(feature_graph, opts)

# # #

choose_stocks = choose_stocks_from_modularity(opts)

print(choose_stocks)

# #

# 测试

# check_modularity_effect(opts)