def __create_output_dir(self):
"""
Internal function that creates the output dir and copies the template file
"""
# Create results directory and create output file from template
dir_name = datetime.datetime.now().strftime("%d%m%Y_%H%M%S")
output_dir = lib_directory_ops.create_dir(self.params['Excel output dir'], dir_name)
assert output_dir != None
new_file = 'output_' + dir_name + '.xlsx'
output_file = lib_path_ops.join_paths(output_dir, new_file)
r = lib_file_ops.copy_file(self.params['Excel template file'], output_file)
self.params['Excel output file'] = output_file
self.write['generation row index'] = 13
assert r != None
def create_dir(root_dir, dir_name):
"""
Function that creates a directory in root_dir
:param root_dir: the directory root
:param dir_name: the name of the directory
:return:
"""
d = None
try:
d = lib_path_ops.join_paths(root_dir, dir_name)
os.mkdir(d)
except OSError:
print("Could not create directory in {}.".format(root_dir))
return d
# Go through lib_sql and protect the functions against potential errors (trying to insert a column name that already exists). The best way to go may be to create function that filters what the user is inputing, rather than protect the functions themselves.
# Make the delete column function (https://www.sqlite.org/faq.html#q6)
#----------------------------------------------------------------------------------------
# IMPORTS
#----------------------------------------------------------------------------------------
import sqlite3 as lib_sql
import lib_path_ops as lib_path
import lib_file_ops as lib_file
import sys
#----------------------------------------------------------------------------------------
# INPUTS
#----------------------------------------------------------------------------------------
root_path = lib_path.get_root_path(lib_path.get_abs_path(""), "./libraries")
database_path = lib_path.join_paths(root_path, "databases")
#----------------------------------------------------------------------------------------
# FUNCTIONS
#----------------------------------------------------------------------------------------
def connect_to_database(database_path, database_name):
"""
Function that connects to a database and returns the connection and cursor objects
"""
try:
database_name = database_name + ".db"
path = lib_path.join_paths(database_path, database_name)
con = lib_sql.connect(path)
cur = con.cursor()
except lib_sql.Error:
import yaml
import sys
import os
#----------------------------------------------------------------------------------------
# PRE-CALCULATIONS
#----------------------------------------------------------------------------------------
# Get inputs
root_dir = os.getcwd()
root_dir = root_dir + '/'
#----------------------------------------------------------------------------------------
# INPUTS
#----------------------------------------------------------------------------------------
try:
input_file = lib_path_ops.join_paths(root_dir, 'inputs/inputs.yaml')
with open(input_file, 'r') as ymlfile:
cfg = yaml.load(ymlfile, Loader=yaml.FullLoader)
activities_dir = cfg['activities path']
week = cfg['week']
print("Loaded inputs successfully.")
except:
print("Failed to load inputs. Exiting...")
sys.exit(1)
#----------------------------------------------------------------------------------------
# MAIN
#----------------------------------------------------------------------------------------
if __name__ == "__main__":
[
def get_p_dic(root_dir):
"""
Function that reads the yaml config files (inputs + api_key) and returns a dictionary with
:param root_dir: directory of execution
:return: dictionary containing all the parameters needed to run the script
"""
# Get inputs from inputs.yaml
try:
input_file = lib_path_ops.join_paths(root_dir, 'inputs/inputs.yaml')
with open(input_file, 'r') as ymlfile:
cfg = yaml.load(ymlfile, Loader=yaml.FullLoader)
filter_p_dic = cfg['filter_params']
backtesting_p_dic = cfg['backtesting_params']
exec_p_dic = cfg['exec_params']
db_cred_file = lib_path_ops.join_paths(root_dir, 'inputs')
db_cred_file = lib_path_ops.join_paths(
db_cred_file, exec_p_dic['Equities DB connection parameters file'])
db_cred_file = db_cred_file.replace('Trading algorithms',
'Data gathering')
with open(db_cred_file, 'r') as ymlfile:
db_cfg = yaml.load(ymlfile, Loader=yaml.FullLoader)
eq_connection_p_dic = db_cfg['Connection parameters']
print("Loaded inputs successfully.")
except:
print("Failed to load inputs. Exiting...")
sys.exit(1)
additional_p_dic = {
"Excel output dir":
lib_path_ops.join_paths(root_dir, 'outputs/'),
"Excel template file":
lib_path_ops.join_paths(
root_dir,
lib_path_ops.join_paths('outputs/',
exec_p_dic['Output template'])),
"Database directory":
lib_path_ops.join_paths(root_dir, 'databases/Euronext')
}
# Start building single p_dic. Start with exec_params
d = exec_p_dic
d.update(backtesting_p_dic)
d.update(additional_p_dic)
d.update(eq_connection_p_dic)
# Convert dates to datetime.date
if d['Start date']:
d['Start date'] = dt.datetime.strptime(str(d['Start date']),
date_format).date()
if d['End date']:
d['End date'] = dt.datetime.strptime(str(d['End date']),
date_format).date()
# Add desired algorithm params to dictionary (Need to convert string to function handle)
f = 'lib_alg.' + d['Algorithm name']
d['Algorithm function'] = eval(f)
# Add desired algorithm parameters
alg_params = d['Algorithm name'] + '_params'
d.update(cfg[alg_params])
# Add filters to dictionary
if d['Filters list'] != []:
# Need to convert strings to function handles
aux = []
for filter in d['Filters list']:
f = 'lib_filters.{}'.format(filter)
aux.append(eval(f))
filter_p_dic['Filters list'] = aux
d.update(filter_p_dic)
return d
def main(p_dic):
"""
Main function that executes all the steps of the analysis in sequence
:param p_dic: dictionary with execution parameters
:return: the 'output.xlsx' file
"""
# Run algorithm
if p_dic['Run algorithm']:
# Create results directory and file (if single file)
dir_name = dt.datetime.now().strftime("%Y%m%d_%H%M%S")
output_dir = lib_directory_ops.create_dir(p_dic['Excel output dir'],
dir_name)
assert output_dir != None
create_file = False
out_prefix = ''
if p_dic['Mode'] == 'Analysis':
out_prefix = 'an'
create_file = True
elif p_dic['Mode'] == 'Backtesting' and p_dic[
'Output file mode'] == 'Single file':
out_prefix = 'btst'
create_file = True
elif p_dic['Mode'] == 'Optimization' and p_dic[
'Output file mode'] == 'Single file':
out_prefix = 'opt'
create_file = True
if create_file:
new_file = out_prefix + '_' + dir_name + '.xlsx'
output_file = lib_path_ops.join_paths(output_dir, new_file)
r = lib_file_ops.copy_file(p_dic['Excel template file'],
output_file)
assert r != None
p_dic['Excel output file'] = output_file
# Initialise tables manager
mngr = Trading_Manager(p_dic)
# For every stock...
n_total_stocks = len(p_dic['Equities list'])
n_stocks_analyzed = 0
conn = lib_pg.get_pg_conn(p_dic['host'], p_dic['database'],
p_dic['user'], p_dic['port'],
p_dic['password'])
for ticker in p_dic['Equities list']:
# Get equity data
# Get module
query = """SELECT "RetrievalPackage" FROM public."Master" WHERE "Ticker" = '{}' AND "TableType" = 'History';""".format(
ticker)
rows = lib_pg.query_pg_database(conn, query)
ticker_mod_name = ""
if len(rows) > 1:
print(
"[WARNING] More than one retrieval package for ticker: {}".
format(ticker))
print("[WARNING] Using package: {}".format(rows[0][0]))
if len(rows) == 1:
ticker_mod_name = rows[0][0]
if ticker_mod_name and lib_db_eq.exists_pg_master_table(
conn, 'History', ticker, ticker_mod_name):
# Get short name
query = """SELECT "ShortName" FROM public."Metadata" WHERE "Ticker" = '{}';""".format(
ticker)
rows = lib_pg.query_pg_database(conn, query)
ticker_sname = rows[0][0]
# Get historic data
columns = '"Date", "Open", "High", "Low", "Close", "Volume"'
query = """SELECT {} FROM public."History-{}-{}";""".format(
columns, ticker_mod_name, ticker)
rows = lib_pg.query_pg_database(conn, query)
#columns = columns.strip(' ')
#columns = columns.split(',')
history_df = pd.DataFrame(
rows,
columns=["Date", "Open", "High", "Low", "Close", "Volume"])
history_df = history_df.set_index('Date')
assert not (history_df.empty)
eq = Stock(ticker, ticker_sname, history_df)
# Determine start and end dates for analysis and dataset
[start_date_ds_obj, end_date_ds_obj
] = lib_general_ops.get_dataset_time_interval(history_df)
# Apply filters only to analysis interval
analyze_ticker = True
filters_function_list = p_dic['Filters list']
if p_dic['Apply filters'] and analyze_ticker:
for filter in filters_function_list:
# if a ticker does not pass a single filter it will not be analyzed
if not filter(eq, start_date_ds_obj, end_date_ds_obj,
p_dic):
analyze_stock = False
print('%s triggered filter <%s>.' %
(eq.get_name(), filter.__name__))
break
# Only for adequate stocks is the trading algorithm applied
if analyze_ticker:
# Start analysis or Backtesting or Optimization....
n_stocks_analyzed = n_stocks_analyzed + 1
out_prefix = ''
create_file = False
if p_dic['Mode'] == 'Analysis':
print('{} is being analyzed...'.format(eq.get_name()))
out_prefix = 'an'
if p_dic['Mode'] == 'Backtesting':
print('{} is being backtested...'.format(
eq.get_name()))
out_prefix = 'btst'
if p_dic['Mode'] == 'Optimization':
print('{} is being backtested through optimization...'.
format(eq.get_name()))
out_prefix = 'opt'
if (p_dic['Mode'] == 'Backtesting'
or p_dic['Mode'] == 'Optimization'
) and p_dic['Output file mode'] == 'Multiple files':
new_file = out_prefix + '_' + eq.get_ticker().replace(
'.', '-') + '_' + dir_name + '.xlsx'
output_file = lib_path_ops.join_paths(
output_dir, new_file)
r = lib_file_ops.copy_file(
p_dic['Excel template file'], output_file)
assert r != None
p_dic['Excel output file'] = output_file
mngr = Trading_Manager(p_dic)
[sdate,
edate] = lib_general_ops.get_analysis_time_interval(
eq.get_dataset(),
date_format,
start_date=p_dic['Start date'],
end_date=p_dic['End date'],
last_n_points=p_dic['Last N days'])
iterations = [1]
if p_dic['Mode'] == 'Optimization':
#iterations = opt.iterations()
pass
for i in iterations:
pop = [1]
if p_dic['Mode'] == 'Optimization':
# pop = opt.population()
pass
for p in pop:
curr_date = sdate
while edate - curr_date >= dt.timedelta(0):
# Run trading algorithm
custom_dic = p_dic['Algorithm function'](
p_dic, eq, curr_date)
# Manage orders and positions
mngr.manage_orders_positions(
custom_dic, eq, curr_date)
# Increment current date
curr_date = eq.get_next_day(curr_date)
del eq
# Write results
if (p_dic['Mode'] == 'Backtesting'
or p_dic['Mode'] == 'Optimization'
) and p_dic['Output file mode'] == 'Multiple files':
mngr.write_results()
del mngr
# Write results
if p_dic['Mode'] == 'Analysis' or p_dic[
'Output file mode'] == 'Single file':
mngr.write_results()
del mngr
# Statistics
if n_total_stocks > 0:
print("\nFinished !\n%s of %s stocks analyzed (%0.1f %%)" %
(str(n_stocks_analyzed), str(n_total_stocks),
float(n_stocks_analyzed) / n_total_stocks * 100))
else:
print("\nFinished ! No stocks analyzed")
return 0
