def all_possible_urls(end_date):
"""Returns urls and saving path for all possible files to download
:param end_date:
:return:
"""
# This is the base url and the file extension
url = AlgoSettings().fxcm_data_path()
store_path = pathlib.Path(AlgoSettings().store_originals_fxcm())
url_suffix = '.csv.gz'
# Set the dates
start_dt = datetime.date(2015, 1, 1)
end_dt = datetime.datetime.strptime(end_date, "%Y-%m-%d").date()
# years
start_year = start_dt.isocalendar()[0]
end_year = end_dt.isocalendar()[0]
# Create dict with all the required urls
urls = {}
for symbol in SYMBOLS:
for yr in range(start_year, end_year + 1):
# All within same year
if start_year == end_year:
start_wk = start_dt.isocalendar()[1]
end_wk = end_dt.isocalendar()[1]
else:
# When more than a year - first year
if yr == start_year:
start_wk = start_dt.isocalendar()[1]
end_wk = datetime.date(yr, 12, 28).isocalendar()[1] + 1
# When more than a year - end year
elif yr == end_year:
start_wk = 1
end_wk = end_dt.isocalendar()[1] + 1
# When more than a year - in between
else:
start_wk = 1
end_wk = datetime.date(yr, 12, 28).isocalendar()[1] + 1
# Construct URLs and saving paths. Save to dictionary
for wk in range(start_wk, end_wk):
data_folder = store_path / symbol / str(yr)
file_name = symbol + '_' + str(yr) + '_' + str(wk) + url_suffix
file_url = '{}/{}/{}/{}{}'.format(url, symbol, yr, wk,
url_suffix)
url_in = {
'url': file_url,
'dir_path': data_folder,
'file_path': data_folder / file_name
}
urls[file_name] = url_in
return urls
def update_all(final_date):
"""Main update function for FXCM files
Download and clean files from server and save then in the appropriate
location. Run from 2014-01-01 as provided by fxcm
:param final_date: date to run to
:return: updated original and clean directories.
"""
log_title('Getting files from FXCM')
urls = definitive_urls(overwrite=False, end_date=final_date)
get_files(urls)
clean_fxcm_originals(
original_dirpath=AlgoSettings().store_originals_fxcm(),
clean_dirpath=AlgoSettings().store_clean_fxcm())
def influx_client(client_type='client', user_type='reader'):
"""Instantiate a connection to the InfluxDB.
:param client_type: 'client' / 'dataframe'
:param user_type: 'reader' / 'writer' / 'admin'
:return:
"""
# Get the configuration info
config = AlgoSettings().influx_config()
# initialize the client
host = config['host']
port = config['port']
user = config['user_' + user_type]
password = config['password_' + user_type]
dbname = config['database']
client = None
try:
if client_type == 'client':
client = InfluxDBClient(host, port, user, password, dbname)
elif client_type == 'dataframe':
client = DataFrameClient(host, port, user, password, dbname)
except (InfluxDBServerError, InfluxDBClientError, KeyError):
logging.exception('Can not connect to database Influxdb.')
raise SystemError
return client
def update_conf_file():
"""Update the logging configuration file with the paths
defined in the CONFIG file
"""
sett = AlgoSettings()
saving_path = pathlib.Path(sett.log_saving_path())
config_file = pathlib.Path(sett.log_configuration())
with open(config_file) as my_file:
doc = yaml.load(my_file)
doc['handlers']['info_file_handler']['filename'] = \
str(saving_path / 'bsk_info.log')
doc['handlers']['error_file_handler']['filename'] = \
str(saving_path / 'bsk_error.log')
with open(config_file, 'w') as my_file:
yaml.dump(doc, my_file)
def setup_logging(default_level=logging.INFO):
"""Setup logging configuration
:param default_level:
:return:
"""
path = AlgoSettings().log_configuration()
path = pathlib.Path(path)
try:
with open(path, 'rt') as my_file:
config = yaml.safe_load(my_file.read())
logging.config.dictConfig(config)
except:
logging.basicConfig(level=default_level)
raise SystemError
def multiple_file_insert():
"""Main function for data insert of multiple files.
:return:
"""
store = pathlib.Path(AlgoSettings().store_clean_fxcm())
time0 = datetime.datetime.now()
log_title("START LOADING MULTIPLE TICK FILES")
load_multiple_tick_files(dir_path=store,
provider='fxcm',
into_table='fx_ticks',
overwrite=False,
validation_type='fast')
time1 = datetime.datetime.now()
logger.info('TOTAL RUNNING TIME WAS: {}'.format(time1 - time0))
def definitive_urls(overwrite, end_date):
"""Remove urls for files already in store if overwrite False
:param overwrite: bol
:param end_date: str
:return: urls dictionary
"""
logger.info('Building urls dictionary')
# Construct all possible URL and saving paths since 2015-01-01 until the
# end date.
possible_urls = all_possible_urls(end_date)
if not overwrite:
# What files are already in store
already_in_store = in_store(AlgoSettings().store_originals_fxcm())
for filepath in already_in_store:
filename = filepath.parts[-1]
if filename in possible_urls:
del possible_urls[filename]
logger.info('Urls dictionary ready')
return possible_urls