def __check_missing_bars(self, df, contract_id, exchange):
# Todo: MAX_GAP_SIZE as parameter
# Download exchange trading calendar, if not yet present
if exchange not in self.__trading_days:
self.__get_trading_calendar(exchange)
# Prepare data for comparison
contract_trading_days = df.index.strftime('%Y-%m-%d').to_list()
exchange_trading_days = self.__trading_days[exchange]
start_date = contract_trading_days[0]
end_date = datetime.today().strftime('%Y-%m-%d')
exchange_trading_days = exchange_trading_days[start_date:end_date]
exchange_trading_days = \
exchange_trading_days.index.strftime('%Y-%m-%d').to_list()
# Find start of earliest gap
MAX_GAP_SIZE = int(get_config_value('quality_check', 'max_gap_size'))
previous_missing_bars = 0
remove_from = ''
for day in exchange_trading_days:
if day not in contract_trading_days:
if previous_missing_bars >= MAX_GAP_SIZE:
remove_from = day
previous_missing_bars += 1
else:
previous_missing_bars = 0
# Return date, from where to delete quotes. -1 if none.
if remove_from != '':
return remove_from
else:
return -1
def __on_tws_error(self, reqId, errorCode, errorString, contract):
"""
Is called from 'ib_insync' as callback on errors and writes error
details to quotes_status in database.
Args:
reqId: Description.
errorCode: Description.
errorString: Description.
contract: Description.
Returns:
Nothing
Raises:
No errors
"""
# Abort receiving if systematical problem is detected
NON_SYSTEMIC_CODES = get_config_value('tws_connector',
'non_systemic_codes')
NON_SYSTEMIC_CODES = list(map(int, NON_SYSTEMIC_CODES))
if errorCode not in NON_SYSTEMIC_CODES:
logging.error(
f"Systemic problem in TWS connection detected. {errorCode}: {errorString}"
)
self.__connection_error = True
# Write error info to contract database, if error is related to contract
if contract is not None:
self.__contract_error_status = errorString
self.__contract_error_code = errorCode
logging.error(
f"Problem for {contract} detected. {errorCode}: {errorString}")
def __check_too_few_quotes(self, df):
# Check overall number of bars
# Decide if contract should be kept
# Return decision result
# Todo: Return barcount
MIN_QUOTES_COUNT = int(
get_config_value('quality_check', 'min_quotes_count'))
if len(df) < MIN_QUOTES_COUNT:
return False
else:
return True
def __check_missing_quotes_at_end(self, df):
# Check for bars missing at end (to presence)
# Decide if contract should be kept
# Return decision result
# Todo: Use trading calendar
# Todo: Return missing bars count
if len(df) == 0:
return False
MAX_MISSING_QUOTES_COUNT = int(
get_config_value('quality_check', 'max_missing_quotes_at_end'))
start_date = str(df.index[-1].date())
end_date = datetime.today().strftime('%Y-%m-%d')
ndays = np.busday_count(start_date, end_date)
if ndays > MAX_MISSING_QUOTES_COUNT:
return False
else:
return True
def test_get_config_value(mock_homepath):
test_config = {
'section_1': {
'option_11': "123",
'option 12!': "-123.45",
'option 12!': "0"
},
'section_2': {
'option_21': "123,456",
'\# Comment': "",
'option 22!': "Lorem Ipsum"
}
}
# Create configparser
parser = configparser.ConfigParser(allow_no_value=True)
# Write test configuration to file
path = Path.home()
if not Path.is_dir(path / ".barbucket"):
Path.mkdir(path / ".barbucket")
path = Path.home() / ".barbucket/config.ini"
parser.read_dict(test_config)
with open(path, 'w') as file:
parser.write(file)
# Read test configuration from file and assert
for section in test_config:
options = test_config[section]
for option in options:
code_value = test_config[section][option]
file_value = cfg.get_config_value(section, option)
if "," in code_value:
assert code_value.split(",") == file_value
else:
assert code_value == file_value
def fetch_historical_quotes(self, universe):
logging.info(f"Fetching historical quotes for universe {universe}.")
# Instanciate necessary objects
universe_db = UniversesDatabase()
tws = Tws()
contracts_db = ContractsDatabase()
quotes_db = QuotesDatabase()
quotes_status_db = QuotesStatusDatabase()
tools = Tools()
# Get config constants
REDOWNLOAD_DAYS = int(get_config_value('quotes', 'redownload_days'))
# Get universe members
contract_ids = universe_db.get_universe_members(universe=universe)
# Setup progress bar
manager = enlighten.get_manager()
pbar = manager.counter(total=len(contract_ids),
desc="Contracts",
unit="contracts")
exiter = GracefulExiter()
tws.connect()
logging.info(f"Connnected to TWS.")
try:
for contract_id in contract_ids:
# Abort, don't process further contracts
if exiter.exit() or tws.has_error():
logging.info("Aborting historical quotes fetching.")
break
# Get contracts data
filters = {'contract_id': contract_id}
columns = ['broker_symbol', 'exchange', 'currency']
contract = contracts_db.get_contracts(
filters=filters, return_columns=columns)[0]
quotes_status = quotes_status_db.get_quotes_status(contract_id)
logging.info(
f"Preparing to fetch hiostorical quotes for {contract['broker_symbol']} on {contract['exchange']}"
)
# Calculate length of requested data
if quotes_status is None:
duration_str = "15 Y"
quotes_from = date.today()
fifteen_years = timedelta(days=5479)
quotes_from -= fifteen_years
quotes_till = date.today().strftime('%Y-%m-%d')
elif quotes_status['status_code'] == 1:
start_date = (quotes_status['daily_quotes_requested_till'])
end_date = date.today().strftime('%Y-%m-%d')
ndays = np.busday_count(start_date, end_date)
if ndays <= REDOWNLOAD_DAYS:
logging.info(
f"Existing data is only {ndays} days old. Contract aborted."
)
pbar.total -= 1
pbar.update(incr=0)
continue
if ndays > 360:
logging.info(
f"Existing data is already {ndays} days old. Contract aborted."
)
pbar.total -= 1
pbar.update(incr=0)
continue
ndays += 6
duration_str = str(ndays) + ' D'
quotes_from = quotes_status['daily_quotes_requested_from']
quotes_till = end_date
else:
logging.info("Contract already has error status. Skipped.")
pbar.total -= 1
pbar.update(incr=0)
continue
# Request quotes from tws
quotes = tws.download_historical_quotes(
contract_id=contract_id,
symbol=contract['broker_symbol'],
exchange=tools.encode_exchange_ib(contract['exchange']),
currency=contract['currency'],
duration=duration_str)
if quotes is not None:
# Inserting quotes into database
logging.info(f"Storing {len(quotes)} quotes to database.")
quotes_db.insert_quotes(quotes=quotes)
# Write finished info to contracts database
quotes_status_db.insert_quotes_status(
contract_id=contract_id,
status_code=1,
status_text="Successful",
daily_quotes_requested_from=quotes_from,
daily_quotes_requested_till=quotes_till)
else:
# Write error info to contracts database
error_code, error_text = tws.get_contract_error()
quotes_status_db.insert_quotes_status(
contract_id=contract_id,
status_code=error_code,
status_text=error_text,
daily_quotes_requested_from=None,
daily_quotes_requested_till=None)
pbar.update()
logging.info(
f"Finished fetching historical quotes for universe {universe}."
)
finally:
tws.disconnect()
logging.info(f"Disconnnected from TWS.")
def connect(self, ):
IP = get_config_value('tws_connector', 'ip')
PORT = int(get_config_value('tws_connector', 'port'))
logging.info(f"Connecting to TWS on {IP}:{PORT}.")
self.__ib.connect(host=IP, port=PORT, clientId=1, readonly=True)