def __init__(self,
temp_data_folder=constants.temp_data_folder,
temp_large_data_folder=constants.temp_large_data_folder,
tickers=None,
data_store=None,
access_control=AccessControl()):
self._temp_data_folder = temp_data_folder
self._temp_large_data_folder = temp_large_data_folder
self._tickers = None
self._util_func = UtilFunc()
self._time_series_ops = TimeSeriesOps()
self._data_store = data_store
self._access_control = access_control
logger = LoggerManager().getLogger(__name__)
if not (os.path.isdir(self._temp_data_folder)):
logger.warning("Temp data folder " + self._temp_data_folder +
" does not exist")
if not (os.path.isdir(self._temp_large_data_folder)):
logger.warning("Temp large data folder " + self._temp_data_folder +
" does not exist")
if tickers is not None:
self._tickers = tickers
def __init__(self,
temp_data_folder=constants.temp_data_folder,
temp_large_data_folder=constants.temp_large_data_folder,
tickers=None,
data_store=constants.dukascopy_data_store,
access_control=AccessControl()):
super(DatabasePopulatorEikon,
self).__init__(temp_data_folder=temp_data_folder,
temp_large_data_folder=temp_large_data_folder,
tickers=tickers,
data_store=data_store,
access_control=access_control)
def upload_trade_data_flat_file(self, sql_database_type=None, trade_data_database_name=None,
csv_sql_table_trade_order_mapping=None,
server_host=None, server_port=None, if_exists_trade_table='replace',
access_control=AccessControl()):
if sql_database_type == 'ms_sql_server':
from tcapy.data.databasesource import DatabaseSourceMSSQLServer as DatabaseSource
username = access_control.ms_sql_server_username; password = access_control.ms_sql_server_password
elif sql_database_type == 'mysql':
from tcapy.data.databasesource import DatabaseSourceMySQL as DatabaseSource
username = access_control.mysql_username; password = access_control.mysql_password
elif sql_database_type == 'sqlite':
from tcapy.data.databasesource import DatabaseSourceSQLite as DatabaseSource
username = access_control.ms_sql_server_username;
password = access_control.ms_sql_server_password
if server_host is None and server_port is None:
database_source = DatabaseSource(trade_data_database_name=trade_data_database_name,
username=username, password=password)
elif server_port is None:
database_source = DatabaseSource(trade_data_database_name=trade_data_database_name,
server_host=server_host,
username=username, password=password)
elif server_host is None:
database_source = DatabaseSource(trade_data_database_name=trade_data_database_name,
server_port=server_port,
username=username, password=password)
else:
database_source = DatabaseSource(trade_data_database_name=trade_data_database_name,
server_host=server_host, server_port=server_port,
username=username, password=password)
for key in csv_sql_table_trade_order_mapping.keys():
database_source.convert_csv_to_table(
csv_sql_table_trade_order_mapping[key], None, key, database_name=trade_data_database_name,
if_exists_table=if_exists_trade_table)
def upload_market_data_flat_file(self, data_vendor='dukascopy', market_data_store='arctic', server_host=None,
server_port=None,
ticker_mkt=['EURUSD', 'GBPUSD', 'AUDUSD', 'NZDUSD', 'USDCAD', 'USDCHF',
'EURNOK', 'EURSEK', 'USDJPY',
'USDNOK', 'USDSEK', 'EURJPY',
'USDMXN', 'USDTRY', 'USDZAR', 'EURPLN'],
csv_folder=None,
if_exists_table='replace',
if_append_replace_ticker='replace',
file_extension='parquet',
read_in_reverse=False, remove_duplicates=False, date_format=None,
plot_back_data=False, access_control=AccessControl(), market_data_database_table=None):
logger = LoggerManager.getLogger(__name__)
# Files dumped by DatabasePopulator look like this
## 'AUDUSD_dukascopy_2016-01-03_22_00_01.868000+00_002016-01-31_23_59_57.193000+00_00.parquet'
csv_file = [x + '_' + data_vendor + '_20*.' + file_extension for x in
ticker_mkt] # assume that ALL TIME IN UTC!
####################################################################################################################
# Load market data
# Create market data store for database and associated data vendor
if market_data_store == 'arctic':
if server_host is None:
server_host = constants.arctic_host
if server_port is None:
server_port = constants.arctic_port
database_source = DatabaseSourceArctic(postfix=data_vendor,
username=access_control.arctic_username,
password=access_control.arctic_password,
server_host=server_host, server_port=server_port)
if market_data_database_table is None:
market_data_database_table = constants.arctic_market_data_database_table
if market_data_store == 'pystore':
database_source = DatabaseSourcePyStore(postfix=data_vendor)
if market_data_database_table is None:
market_data_database_table = constants.pystore_market_data_database_table
if market_data_store == 'influxdb':
if server_host is None:
server_host = constants.influxdb_host
if server_port is None:
server_port = constants.influxdb_port
database_source = DatabaseSourceInfluxDB(postfix=data_vendor,
username=access_control.influxdb_username,
password=access_control.influxdb_password,
server_host=server_host, server_port=server_port)
if market_data_database_table is None:
market_data_database_table = constants.influxdb_market_data_database_table
if market_data_store == 'kdb':
if server_host is None:
server_host = constants.kdb_host
if server_port is None:
server_port = constants.kdb_port
database_source = DatabaseSourceKDB(postfix=data_vendor,
username=access_control.kdb_username,
password=access_control.kdb_password,
server_host=server_host, server_port=server_port)
if market_data_database_table is None:
market_data_database_table = constants.kdb_market_data_database_table
if csv_folder is None:
csv_folder = constants.test_data_folder
# This relies on you have market data stored in Parquet/H5/CSV files already (eg. by downloading from DukasCopy)
# note: whilst free FX data can be used for testing (in particular for generating randomised trades),
# you may to choose other high frequency quality data for actual benchmark
csv_market_data = [os.path.join(csv_folder, x) for x in csv_file]
# For each ticker, read in the H5/CSV file and then dump into tick database
# Potentionally, we can thread this?
for i in range(0, len(ticker_mkt)):
ticker = ticker_mkt[i]
csv_file = csv_market_data[i]
# On the second time through the loop, we make sure to append to table
# otherwise will keep overwriting!
if if_exists_table == 'replace':
if i >= 1:
if_exists_table = 'append'
else:
if_exists_table = 'replace'
database_source.convert_csv_to_table(csv_file, ticker, market_data_database_table,
if_exists_table=if_exists_table,
remove_duplicates=remove_duplicates,
if_exists_ticker=if_append_replace_ticker, date_format=date_format,
read_in_reverse=read_in_reverse)
# It is worth plotting the data to check validity sometimes (make sure you choose appropriate start/finish dates
# loading a *very* large tick history into memory will result in your computer running out of memory
if plot_back_data:
from chartpy import Chart
import datetime
import pandas as pd
df = database_source.fetch_market_data(start_date='01 Jan 2000',
finish_date=datetime.datetime.utcnow(),
ticker=ticker)
df = pd.DataFrame(df.resample('5min').mean())
if 'mid' not in df.columns:
df['mid'] = (df['bid'] + df['ask']) / 2.0
df = pd.DataFrame(df['mid'])
Chart(engine='plotly').plot(df)
print(df)
logger.info("Finished uploading data to " + market_data_store)