if data_vendor == 'ncfx':
from tcapy.data.databasepopulator import DatabasePopulatorNCFX as DatabasePopulator
tickers = constants.ncfx_tickers
elif data_vendor == 'dukascopy':
from tcapy.data.databasepopulator import DatabasePopulatorDukascopy as DatabasePopulator
tickers = constants.dukascopy_tickers
# Example of manually specifying _tickers
# tickers = {'EURUSD' : 'EURUSD', 'GBPUSD': 'GBPUSD', 'USDCAD': 'USDCAD', 'NZDUSD': 'NZDUSD', 'USDCHF' : 'USDCHF',
# 'USDJPY' : 'USDJPY'}
db_populator = DatabasePopulator(
temp_data_folder=temp_data_folder,
temp_large_data_folder=temp_large_data_folder,
tickers=tickers)
# Writes a CSV/Parquet to disk from data vendor (does not attempt to write anything to the database)
# Will also dump temporary HDF5 files to disk (to avoid reloading them)
msg, df_dict = db_populator.download_to_csv(
start_date_csv,
finish_date_csv,
tickers,
split_size=split_size,
csv_folder=csv_folder,
return_df=False,
remove_duplicates=False,
write_large_csv=write_large_csv,
write_large_hdf5_parquet=write_large_hdf5_parquet)
# }
chunk_int_min_dict = {
'dukascopy': None,
'ncfx': 60
} # number of minutes to download from data vendor (eg. 5 minutes)
########################################################################################################################
folder = Constants().test_data_harness_folder
#### change for your data vendor
data_vendor_name_list = ['ncfx', 'dukascopy']
database_populator_dict = {
'dukascopy': DatabasePopulatorDukascopy(),
'ncfx': DatabasePopulatorNCFX()
}
database_source_dict = {
'dukascopy': DatabaseSourceDukascopy(),
'ncfx': DatabaseSourceNCFX()
}
if constants.ncfx_url is None:
data_vendor_name_list.remove('ncfx')
####
trade_order_list = ['trade_df', 'order_df']
sql_trade_order_mapping = OrderedDict([
('trade_df',
web_proxies = {'https' : None}
# web_proxies = {
# 'http' : "http://127.0.0.1:8080",
# 'https' : "https://127.0.0.1:7000",
# }
chunk_int_min_dict = {'dukascopy' : None, 'ncfx' : 60} # number of minutes to download from data vendor (eg. 5 minutes)
########################################################################################################################
folder = Constants().test_data_harness_folder
#### Change for your data vendor
data_vendor_name_list = ['dukascopy'] # ['ncfx', 'dukascopy']
database_populator_dict = {'dukascopy' : DatabasePopulatorDukascopy(), 'ncfx' : DatabasePopulatorNCFX()}
database_source_dict = {'dukascopy' : DatabaseSourceDukascopy(), 'ncfx' : DatabaseSourceNCFX()}
if constants.ncfx_url is None and 'ncfx' in data_vendor_name_list:
data_vendor_name_list.remove('ncfx')
if constants.ncfx_url is not None and 'ncfx' in data_vendor_name_list:
if len(constants.ncfx_url) < 10:
data_vendor_name_list.remove('ncfx')
invalid_start_date = '01 Jan 1999'
invalid_finish_date = '01 Feb 1999'
use_multithreading = False
def test_fetch_market_data_from_data_vendor():
if data_vendor == 'ncfx':
from tcapy.data.databasepopulator import DatabasePopulatorNCFX as DatabasePopulator
tickers = constants.ncfx_tickers
data_store = 'arctic-ncfx'
elif data_vendor == 'dukascopy':
from tcapy.data.databasepopulator import DatabasePopulatorDukascopy as DatabasePopulator
tickers = constants.dukascopy_tickers
data_store = 'arctic-dukascopy'
db_populator = DatabasePopulator(
temp_data_folder=temp_data_folder,
temp_large_data_folder=temp_large_data_folder,
tickers=tickers,
data_store=data_store)
# The first time we are downloading data from the data vendor
if FIRST_DOWNLOAD:
# WARNING: for very first time we download choose the specified dates
# you may need to change the delete cached files temporarily on disk
msg = db_populator.download_from_external_source(
append_data=True,
start_date=start_date,
finish_date=finish_date,
remove_duplicates=remove_duplicate_follow_on,
delete_cached_files=delete_cached_files,
write_to_disk_db=True,
if_exists_ticker='replace',