def get_futures_daily_close_data(tickers: list,
start_date: datetime = None,
end_date: datetime = None) -> pd.DataFrame:
"""
Returns a DataFrame with the daily close of futures between start and end dates. N/A are rolled forward if they
exists before the last observation date of the resp. futures contract.
:param tickers: list of strings
:param start_date: datetime
:param end_date: datetime
:return: DataFrame
"""
# get the raw data from the financial database
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
raw_futures_data = fin_db.get_close_price_df(tickers, start_date, end_date)
# clean the data by rolling N/A forward
cleaned_futures_data = raw_futures_data.fillna(method='ffill')
# get last observation date per ticker
ticker_last_obs_date_dict = fin_db.get_ticker_underlying_attribute_dict(
tickers, Underlying.latest_observation_date_with_values)
# loop through each column and set each row to N/A if it is after the last observation date for the resp. ticker
for col_i in range(cleaned_futures_data.shape[1]):
last_obs_date = ticker_last_obs_date_dict[list(cleaned_futures_data)
[col_i]]
try:
last_obs_date_index = cleaned_futures_data.index.get_loc(
last_obs_date)
except KeyError: # in case when the last observation period is after end_date
last_obs_date_index = cleaned_futures_data.shape[0]
cleaned_futures_data.iloc[last_obs_date_index + 1:, col_i] = np.nan
return cleaned_futures_data
def get_price(self):
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
price_info = ''
if self.get_currency() is not None:
price_info += '_' + self.get_currency().upper()
if self.total_return_combo.get() == 'No':
result_df = fin_db.get_close_price_df(
tickers=self.ticker_list,
start_date=self.get_start_date(),
end_date=self.get_end_date(),
currency=self.get_currency())
result_df = self.handle_nan_df(result_df)
self.parent.result_df_dict.update(
{'close_price' + price_info: result_df})
else:
result_df = fin_db.get_total_return_df(
tickers=self.ticker_list,
start_date=self.get_start_date(),
end_date=self.get_end_date(),
currency=self.get_currency(),
withholding_tax=self.get_div_tax())
result_df = self.handle_nan_df(result_df)
self.parent.result_df_dict.update(
{'total_return_price' + price_info: result_df})
logger.info('Done with loading price!')
self.cancel()
def __init__(self,
tickers: {str, list, tuple},
start=None,
end=None,
periods=None,
freq=None,
observation_calendar: pd.DatetimeIndex = None):
if com.count_not_none(start, end, periods, freq) != 0:
self._observation_calendar = pd.date_range(start, end, periods,
freq)
else:
if observation_calendar is None:
raise ValueError(
'Need to specify observation_calendar or the parameters of an observation calendar i.e'
'start, end, periods, freq')
elif observation_calendar.is_monotonic_increasing:
self._observation_calendar = observation_calendar
else:
raise ValueError(
'observation_calendar needs to be an instance of a DatatimeIndex object that is '
'monotonic increasing')
self.tickers = tickers
self._financial_database_handler = FinancialDatabase(
__MY_DATABASE_NAME__)
self._filter_has_been_applied = False
self._filter_desc_list = []
def __init__(self, signal_df: pd.DataFrame,
max_instrument_weight: {float, None},
min_instrument_weight: {float, None}):
Weight.__init__(self,
signal_df=signal_df,
max_instrument_weight=max_instrument_weight,
min_instrument_weight=min_instrument_weight)
self._financial_database_handler = FinancialDatabase(
__MY_DATABASE_NAME__)
def __init__(self, tickers: {str,
list}, observation_calendar: pd.DatetimeIndex,
eligibility_df: pd.DataFrame):
Signal.__init__(self,
tickers=tickers,
observation_calendar=observation_calendar,
eligibility_df=eligibility_df)
self._financial_database_handler = FinancialDatabase(
__MY_DATABASE_NAME__)
def get_volume(self):
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
result_df = fin_db.get_volume_df(tickers=self.ticker_list,
start_date=self.get_start_date(),
end_date=self.get_end_date())
result_df = self.handle_nan_df(result_df)
self.parent.result_df_dict.update({'volume': result_df})
logger.info('Done with loading volume!')
self.cancel()
def __init__(self,
max_value: float,
min_value: float,
weight_obs_lag: int = 1,
scaling_factor_lag: int = 0,
avg_smoothing_lag: int = 1):
super().__init__(max_value=max_value,
min_value=min_value,
weight_obs_lag=weight_obs_lag,
scaling_factor_lag=scaling_factor_lag,
avg_smoothing_lag=avg_smoothing_lag)
self._financial_db_handler = FinancialDatabase(__MY_DATABASE_NAME__)
def get_dividend(self):
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
price_info = ''
if self.get_currency() is not None:
price_info += '_' + self.get_currency().upper()
result_df = fin_db.get_dividend_df(tickers=self.ticker_list,
start_date=self.get_start_date(),
end_date=self.get_end_date(),
currency=self.get_currency())
result_df = self.handle_nan_df(result_df)
self.parent.result_df_dict.update({'dividend' + price_info: result_df})
logger.info('Done with loading dividend!')
self.cancel()
def get_underlying_data(self):
if any(selected_var.get() == 1 for selected_var in self.int_vars):
chosen_attributes = [
attribute for i, attribute in enumerate(self.attribute_list)
if self.int_vars[i].get()
]
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
result_df = fin_db.get_underlying_data(self.ticker_list,
chosen_attributes)
self.parent.result_df_dict.update({'underlying_data': result_df})
logger.info('Done with loading underlying data!')
self.cancel()
else:
msg.showinfo(
'Warning',
"Please select an underlying attribute e.g. 'sector' or 'currency'."
)
def run_filter_click(self):
if self.filter_dict == {}: # empty filter
msg_when_running_without_filter = msg.askquestion(
'Warning',
'Are you sure you want to ' +
HandleUnderlyingWindow.chose_action.lower().split()[0] +
' all tickers in the database?',
icon='warning')
# self.master.lift() # window moves backward for some reason ...
if msg_when_running_without_filter == 'no':
return
else:
pass
fin_db_handler = FinancialDatabase(__MY_DATABASE_NAME__)
tickers = fin_db_handler.get_ticker(self.filter_dict)
self.result = tickers
self.cancel()
def basket_prices(self,
start_date: {date, datetime} = None,
end_date: {date, datetime} = None,
forward_fill_na: bool = True):
logger.debug('Get basket price.')
financial_database_handler = FinancialDatabase(__MY_DATABASE_NAME__,
False)
tickers = self.investment_universe.get_eligible_tickers()
if self.total_return:
price = financial_database_handler.get_total_return_df(
tickers, start_date, end_date, self.dividend_tax,
self.currency)
else:
price = financial_database_handler.get_close_price_df(
tickers, start_date, end_date, self.currency)
if forward_fill_na:
price.fillna(inplace=True, method='ffill')
return price
def get_investment_universe(underlying_code: str,
month_codes: list,
start_date: datetime = None,
end_date: datetime = None) -> list:
"""
Returns a list of futures tickers based on the given monthly codes and underlying code
:param underlying_code: str e.g. 'GC' gives gold futures
:param month_codes: list of strings e.g. 'M' corresponds to june and 'Q' to august
:param start_date: datetime only picks contracts where the latest observation date is after this date
:param end_date: datetime only picks contracts where the oldest observation date is before this date
:return: list of strings
"""
# handle inputs
if start_date is None:
start_date = datetime(1950, 1, 1)
if end_date is None:
end_date = datetime.today() + BDay(1) # next business day from today
# combine the underlying code (e.g. 'GC' for gold) and the month codes (e.g. 'M' and 'Q')
contract_codes = [
underlying_code.upper() + month_code.upper()
for month_code in month_codes
]
# find eligible futures contracts in the database by filtering based on dates and contract codes
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
query_futures_tickers = fin_db.session.query(Underlying.ticker).filter(
and_(
or_(*[
Underlying.ticker.like(contract_code + '%')
for contract_code in contract_codes
]), Underlying.latest_observation_date_with_values > start_date,
Underlying.oldest_observation_date < end_date,
Underlying.underlying_type == 'FUTURE')).order_by(
Underlying.description.asc()).all()
# store the resulting tickers in a list
futures_tickers = [tup[0] for tup in query_futures_tickers]
if not len(futures_tickers):
raise ValueError(
'No tickers could be found in the database.\nDatabase: {}\nUnderlying code: {}\nMonth code: %s'
.format(__MY_DATABASE_NAME__, underlying_code.upper()) %
', '.join(month_codes))
return futures_tickers
def get_expiry_date_dict(tickers: list) -> dict:
"""
Return a dictionary: keys = tickers (str), values = expiry dates (datetime)
:param tickers: list of strings
:return: dictionary
"""
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
asset_class = fin_db.get_ticker_underlying_attribute_dict(
tickers, Underlying.underlying_type)
desc = fin_db.get_ticker_underlying_attribute_dict(tickers,
Underlying.description)
result_dict = {}
for ticker in tickers:
if asset_class[ticker] == 'FUTURE':
expiry_desc = desc[ticker]
result_dict.update({
ticker:
datetime.strptime(expiry_desc.split()[1], '%Y-%m-%d')
})
else:
logger.warning(
'{} is of type {} and does not have a expiry date.'.format(
ticker, asset_class[ticker]))
return result_dict
def apply_result(self):
if self.result is not None:
action = self.parent.action_combo.get()
data_source = self.parent.data_source_combo.get()
# initialize a database handler used to add and refresh data
batch_size = None
if data_source == data_source_list[0]:
batch_size = 15
fin_db = YahooFinanceFeeder(__MY_DATABASE_NAME__)
elif data_source == data_source_list[1]:
batch_size = 100
fin_db = BloombergFeeder(__MY_DATABASE_NAME__, bbg_echo=False)
else:
fin_db = FinancialDatabase(__MY_DATABASE_NAME__)
if action == 'Add underlying':
counter = 1
list_of_ticker_list = list_grouper(self.result, batch_size)
tickers_not_available = []
for ticker_sub_list in list_of_ticker_list:
progression_bar(counter, len(list_of_ticker_list))
try:
fin_db.add_underlying(ticker_sub_list)
except ValueError:
logger.warning(
'One ticker is not available in {}.'.format(
data_source))
for ticker in ticker_sub_list:
try:
fin_db.add_underlying(ticker)
except ValueError:
logger.warning(
'{} does not exist as a ticker on {}.'.
format(ticker, data_source))
tickers_not_available.append(ticker)
counter += 1
logger.info('Done with adding underlying(s) to database.')
if len(tickers_not_available) > 0:
logger.warning(
'{} ticker(s) could not be added.\nTicker(s): %s'.
format(len(tickers_not_available)) %
', '.join(tickers_not_available))
elif action == 'Refresh underlying':
counter = 1
list_of_ticker_list = list_grouper(self.result, 100)
for ticker_sub_list in list_of_ticker_list:
progression_bar(counter, len(list_of_ticker_list))
fin_db.refresh_data_for_tickers(ticker_sub_list)
counter += 1
logger.info(
'Done with refreshing underlying(s) in the database.')
elif action == 'Delete underlying':
msg_before_delete = msg.askquestion(
'Warning',
'Are you sure you want to delete {} ticker(s) from the '
'database?'.format(len(self.result)),
icon='warning')
if msg_before_delete == 'no':
return
else:
fin_db.delete_underlying(self.result)
logger.info('Done with deleting underlying(s) from database.')
elif action == 'Download data':
DataRetrievalWindow(self,
title=action,
ticker_list=self.result)
else:
msg.showinfo(
'Action not executable',
"The action '{}' has not been implemented yet".format(
action))
else:
raise ValueError('No tickers selected (self.result is None)')