def opt_func_base(stock_name, start_day_index, end_day_index, X_list):
if len(X_list.shape) == 2:
X_list = X_list[0]
optimize_result = OptimizeResult()
cache_filename = get_cache_filename(stock_name, start_day_index, end_day_index)
if os.path.isfile(cache_filename):
optimize_result.load(cache_filename)
#print("stock worm cache loaded, size={}".format(optimize_result.get_size()))
else:
print("cannot find file cache:{}, will create new cache.".format(cache_filename))
print("Checking: {}".format(X_list))
hmm_model = HmmModel(stock_name)
total_profit, profit_daily_avg = hmm_model.train(X_list, start_day_index, end_day_index)
if total_profit == -1:
print("Training failed.")
return total_profit
print("Finished, total_profit:{}".format(total_profit))
strategy_features = hmm_model.get_strategy_features()
optimize_result.insert_result(X_list, strategy_features + [total_profit, profit_daily_avg])
optimize_result.save(cache_filename)
return profit_daily_avg
def search_worms(self,
start_day_index,
end_day_index,
max_iter=300,
is_test=False,
search_strategy=False):
if is_test == True:
mixed_domain = self.mixed_domain_test
else:
mixed_domain = self.mixed_domain
self.optimize_result = OptimizeResult(result_column_index=15)
stock_worm_cache_file = self.get_stockworm_cache_file(
start_day_index, end_day_index)
if os.path.isfile(stock_worm_cache_file):
self.optimize_result.load(stock_worm_cache_file)
print("stock worm cache loaded, size={}".format(
self.optimize_result.get_size()))
else:
print("cannot find file cache:{}, will create new cache.".format(
stock_worm_cache_file))
self.stock_worm_cache_file = stock_worm_cache_file
strategy_cache_file = self.get_strategy_cache_file(
start_day_index, end_day_index)
trade_strategy_factory = TradeStrategyFactory(
cache_file=strategy_cache_file)
if os.path.isfile(strategy_cache_file) and search_strategy == False:
print("find strategy_cache:{}, loading...".format(
strategy_cache_file))
strategy_list = trade_strategy_factory.create_from_file(
strategy_cache_file, NUM_STRATEGIES)
else:
if search_strategy == True:
print("search_strategy is True, searching strategies again...")
else:
print("cannot find strategy cache:{}, generating...".format(
strategy_cache_file))
data = load_strategy_input_data(self.stock_index, start_day_index,
end_day_index)
# the input data: timestamp, value, and price.
strategy_list = trade_strategy_factory.create_trade_strategies(
data, 5)
opt_func = partial(self.opt_func, strategy_list, start_day_index,
end_day_index)
opt_handler = GPyOpt.methods.BayesianOptimization(
f=opt_func, # Objective function
domain=mixed_domain, # Box-constraints of the problem
initial_design_numdata=30, # Number data initial design
acquisition_type='EI', # Expected Improvement
exact_feval=True,
maximize=True) # True evaluations, no sample noise
opt_handler.run_optimization(max_iter, eps=0)
def __init__(self, cache_file=None, n_max_trades_per_day=4, slippage=0, courtage=0, is_future=False):
self.n_max_trades_per_day = n_max_trades_per_day
self.slippage = slippage
self.courtage = courtage
self.is_future = is_future
self.trade_strategy = None
# load the initial data file
self.optimize_result = OptimizeResult()
self.cache_file = cache_file
if cache_file is not None:
self.optimize_result.load(cache_file)
return
def create_from_file(self, filename, n_number):
optimize_result = OptimizeResult(result_column_index=-1)
optimize_result.load(filename)
data = optimize_result.get_best_results(n_number)
trade_strategy_list = []
for i in range(n_number):
X_list = data[i, :4]
trade_strategy_list.append(
TradeStrategy(X_list, self.n_max_trades_per_day, self.slippage,
self.courtage))
return trade_strategy_list
def __init__(self,
cache_file=None,
n_max_trades_per_day=4,
slippage=0.00015,
courtage=0):
self.n_max_trades_per_day = n_max_trades_per_day
self.slippage = slippage
self.courtage = courtage
self.trade_strategy = None
# load the initial data file
self.optimize_result = OptimizeResult(result_column_index=-1)
if cache_file is not None:
self.cache_file = cache_file
self.optimize_result.load(cache_file)
return
def update_worms_from_cache(self, n_number, start_day_index,
end_day_index):
optimize_result = OptimizeResult()
stockworm_cache_file = self.get_stockworm_cache_file(
start_day_index, end_day_index)
optimize_result.load(stockworm_cache_file)
top_worms = optimize_result.get_best_results(n_number)
assert (len(top_worms) == n_number)
for i in range(n_number):
features = top_worms[i, :15]
strategy_features = top_worms[i, 15:21]
md5 = top_worms[i, -1]
model_save_path = self.get_model_save_path(start_day_index,
end_day_index, md5)
new_worm = StockWorm(self.stock_name,
self.stock_id,
self.npy_files_path,
model_save_path,
is_future=self.is_future,
slippage=self.slippage)
if os.path.isdir(model_save_path) and new_worm.load() == True:
pass
else:
total_profit, profit_daily, errors_daily = new_worm.init(
features,
start_day_index,
end_day_index,
strategy_features=strategy_features)
new_worm.save()
print("training finished for model {}, total_profit:{}".format(
i, total_profit))
new_worm.report()
testing_total_profit, testing_profit_daily, n_data_appended = new_worm.test(
)
if n_data_appended > 0:
print(
"testing finished for model {}, total_profit:{} in {} days, new data for {} days appended"
.format(i, testing_total_profit, len(testing_profit_daily),
n_data_appended))
new_worm.save()
new_worm.report()
self.worm_list.append(new_worm)
def update_worms_from_cache(self, n_number, start_day_index,
end_day_index):
optimize_result = OptimizeResult(result_column_index=15)
stockworm_cache_file = self.get_stockworm_cache_file(
start_day_index, end_day_index)
optimize_result.load(stockworm_cache_file)
top_worms = optimize_result.get_best_results(n_number)
trade_strategy_factory = TradeStrategyFactory()
strategy_cache_file = self.get_strategy_cache_file(
start_day_index, end_day_index)
strategy_list = trade_strategy_factory.create_from_file(
strategy_cache_file, NUM_STRATEGIES)
assert (len(top_worms) == n_number)
swarm_path = self.get_swarm_path(start_day_index, end_day_index)
for i in range(n_number):
features = top_worms[i, :13]
features_str = self.get_parameter_str(features)
model_save_path = os.path.join(swarm_path, "models",
md5(features_str))
new_worm = StockWorm(self.stock_index, self.npy_files_path,
model_save_path)
if os.path.isdir(model_save_path) and new_worm.load() == True:
pass
else:
total_profit, profit_daily, errors_daily = new_worm.init(
features, strategy_list, start_day_index, end_day_index)
new_worm.save()
print("training finished for model {}, total_profit:{}".format(
i, total_profit))
testing_total_profit, testing_profit_daily, n_data_appended = new_worm.test(
)
if n_data_appended > 0:
print(
"testing finished for model {}, total_profit:{} in {} days, new data for {} days appended"
.format(i, testing_total_profit, len(testing_profit_daily),
n_data_appended))
new_worm.save()
self.worm_list.append(new_worm)
def search_worms(self,
start_day_index,
end_day_index,
max_iter=300,
is_test=False):
if is_test == True:
mixed_domain = self.mixed_domain_test
else:
mixed_domain = self.mixed_domain
if self.is_future == True:
mixed_domain = self.mixed_domain_future
self.optimize_result = OptimizeResult()
stock_worm_cache_file = self.get_stockworm_cache_file(
start_day_index, end_day_index)
create_if_not_exist(os.path.dirname(stock_worm_cache_file))
if os.path.isfile(stock_worm_cache_file):
self.optimize_result.load(stock_worm_cache_file)
print("stock worm cache loaded, size={}".format(
self.optimize_result.get_size()))
else:
print("cannot find file cache:{}, will create new cache.".format(
stock_worm_cache_file))
self.stock_worm_cache_file = stock_worm_cache_file
opt_func = partial(self.opt_func, start_day_index, end_day_index)
opt_handler = GPyOpt.methods.BayesianOptimization(
f=opt_func, # Objective function
domain=mixed_domain, # Box-constraints of the problem
initial_design_numdata=30, # Number data initial design
acquisition_type='EI', # Expected Improvement
exact_feval=True,
maximize=True) # True evaluations, no sample noise
opt_handler.run_optimization(max_iter, eps=0)
import sys
import os.path
sys.path.append("../")
from util import *
from optimizeresult import OptimizeResult
from pathlib import Path
stock_data_path = get_stock_data_dir()
# This is to get the directory that the program
# is currently running in.
for filename in Path(stock_data_path).glob('**/stockworm_cache.txt'):
optimize_result = OptimizeResult(result_column_index=-2)
optimize_result.load(filename)
print(optimize_result.get_n_columns())
if (optimize_result.get_n_columns() == 24):
print("{} has been already migrated.".format(filename))
continue
print('migrating {}'.format(filename))
assert(optimize_result.get_n_columns() == 23)
optimize_result.add_column(14, -1)
assert(optimize_result.get_n_columns() == 24)
optimize_result.save(filename)
)
sys.exit()
stock_name = sys.argv[1]
stock_id = get_stock_id_by_name(stock_name)
training_start_day_index = int(sys.argv[2])
training_end_day_index = int(sys.argv[3])
number = 10
if len(sys.argv) == 5:
number = int(sys.argv[4])
swarm_dir = get_swarm_dir(stock_name, stock_id, training_start_day_index,
training_end_day_index)
strategy_file = os.path.join(swarm_dir, 'stockworm_cache.txt')
worm_results = OptimizeResult(-1)
worm_results.load(strategy_file)
print("Top 10 Worms in {} results for {}: swarm: {}-{}".format(
worm_results.get_size(), stock_name, training_start_day_index,
training_end_day_index))
worm_results.get_best_results(number)
columns = [
'n_neurons', 'learning_rate', 'num_layers', 'rnn_type', 'learning_period',
'prediction_period', 'n_repeats', 'beta', 'ema', 'time_format',
'volume_input', 'use_centralized_bid', 'split_daily_data', 'is_stateful',
'ref_stock_id', 'buy_threshold', 'sell_threshold', 'stop_gain',
'stop_gain', 'skip_at_beginning', 'value_ma', 'total_profit', 'days',
'avg_error', 'avg_profit'
]
print("Columns:")
if len(sys.argv) < 5:
print(
"usage: python3 search-worms.py stock_name, stock_index traning_start_day_index, training_end_day_index"
)
sys.exit()
stock_name = sys.argv[1]
stock_index = int(sys.argv[2])
training_start_day_index = int(sys.argv[3])
training_end_day_index = int(sys.argv[4])
swarm_dir = get_swarm_dir(stock_name, stock_index, training_start_day_index,
training_end_day_index)
strategy_file = os.path.join(swarm_dir, 'strategy_cache.txt')
result_strategies = OptimizeResult(-1)
result_strategies.load(strategy_file)
print("Top 10 Strategies in {} results for {}: swarm: {}-{}".format(
result_strategies.get_size(), stock_name, training_start_day_index,
training_end_day_index))
result_strategies.get_best_results(10)
strategy_file = os.path.join(swarm_dir, 'stockworm_cache.txt')
worm_results = OptimizeResult(-2)
worm_results.load(strategy_file)
print("Top 10 Worms in {} results for {}: swarm: {}-{}".format(
worm_results.get_size(), stock_name, training_start_day_index,
training_end_day_index))
worm_results.get_best_results(10)
print(
"usage: python3 update-models.py stock_name, start_day_index end_day_index [number=1]"
)
sys.exit()
stock_name = sys.argv[1]
start_day_index = int(sys.argv[2])
end_day_index = int(sys.argv[3])
if len(sys.argv) == 5:
number = int(sys.argv[4])
else:
number = 1
cache_file = get_cache_filename(stock_name, start_day_index, end_day_index)
optimize_result = OptimizeResult()
optimize_result.load(cache_file)
top_worms = optimize_result.get_best_results(number)
for i in range(len(top_worms)):
features = top_worms[i, :6]
strategy_features = top_worms[i, 6:12]
md5 = top_worms[i, -1]
model_path = get_model_path(stock_name, start_day_index, end_day_index)
save_path = os.path.join(model_path, md5)
hmm_model_filename = os.path.join(save_path, "hmm_model.pkl")
hmm_model = None
if os.path.isfile(hmm_model_filename):
print("file: {} exists, load it.".format(hmm_model_filename))
with open(hmm_model_filename, "rb") as file:
hmm_model = pickle.load(file)