def place_order(action: str, symbol: str, quantity: float, quantity_type: str):
'''
Calls the https://api.tdameritrade.com/v1/accounts/{ORDER_ID}/orders API and
places a simple (single leg) Market order (BUY/SELL) valid through the day for the specified
symbol and dollar amount.
Returns the order id as extracted from the location header. If the order ID
cannot be found, it will return a the two character tag which was randomly
generated during this call
'''
if quantity_type not in ['SHARES', 'DOLLARS', 'ALL_SHARES']:
raise ValidationError(
"Invalid quantity type supplied: %s" % quantity_type, None)
if (action) not in ('BUY', 'SELL'):
raise ValidationError("Invalid trading actions supplied: %s" % action,
None)
td_account_id = get_credentials()[0]
tag = generate_tag()
order = {
"orderType":
"MARKET",
"session":
"NORMAL",
"duration":
"DAY",
"orderStrategyType":
"SINGLE",
"orderLegCollection": [{
"instruction": action,
"quantity": quantity,
"quantityType": quantity_type,
"instrument": {
"assetType": "EQUITY",
"symbol": symbol
}
}],
"tag":
tag
}
headers = request(
'POST',
'https://api.tdameritrade.com/v1/accounts/%s/orders' % td_account_id,
None, order)[0]
# fetch the order id from the headers, This is where it's stored
# https://api.tdameritrade.com/v1/accounts/{ACCT_ID}/orders/{ORDER_ID}'
# --> {ORDER_ID}
try:
order_id = str(headers['Location'].rsplit('/', 1)[-1])
except:
order_id = tag
return order_id
def get_fiscal_year_period(year: int, extend_by_days: int):
"""
returns the first and last day of a year's fiscal period.
For example:
2018 -> ('2018-01-01', '2019-12-31')
the end date can be extended by "extend_by_days"
For example:
(2018, 10) -> ('2018-01-01', '2019-01-10')
Parameters
----------
year : int
the fiscal year in question
extend_by_days : int
number of days by which to extend the fiscal period.
This is done to account for the fact that financial statements may be submitted
shortly after the fiscal period ends
Returns
-----------
A tuple of strings containing the start and end date of the fiscal period
"""
if year < 2000:
raise ValidationError("Invalid Date. Must be >= 2000", None)
if extend_by_days < 0 or extend_by_days > 350:
raise ValidationError("Invalid extend_by_days. Must between 0 and 350",
None)
start = datetime(year, 1, 1)
end = datetime(year, 12, 31) + timedelta(days=extend_by_days)
return (date_to_string(start), date_to_string(end))
def __init__(self, ticker_list: list, analysis_period: str,
current_price_date: date, output_size: int):
"""
Initializes the strategy given the ticker list, analysis period
and output size.
The period is used to determine the range of financial data required
to perform the analysis, while the output size will limit the number
of securities that are recommended by the strategy.
Parameters
------------
ticker_list : list of tickers to be included in the analisys
analysis_period: The analysis period as a string. E.g. '2020-06'
output_size : number of recommended securities that will be returned
by this strategy
"""
if ticker_list == None or len(ticker_list.ticker_symbols) < 2:
raise ValidationError(
"Ticker List must contain at least 2 symbols", None)
if output_size == 0:
raise ValidationError("Output size must be at least 1", None)
try:
self.analysis_period = pd.Period(analysis_period, 'M')
except Exception as e:
raise ValidationError("Could not parse Analysis Period", e)
self.ticker_list = ticker_list
self.output_size = output_size
if (current_price_date == None):
business_date = self.current_price_date = util.get_business_date(
constants.BUSINESS_DATE_DAYS_LOOKBACK,
constants.BUSINESS_DATE_CUTOVER_TIME)
self.current_price_date = business_date
else:
self.current_price_date = current_price_date
(self.analysis_start_date,
self.analysis_end_date) = intrinio_util.get_month_period_range(
self.analysis_period)
if (self.analysis_start_date > self.current_price_date):
raise ValidationError(
"Price Date: [%s] must be greater than the Analysis Start Date [%s]"
% (self.current_price_date, self.analysis_start_date), None)
if (self.analysis_end_date > self.current_price_date):
logging.debug("Setting analysis end date to 'today'")
self.analysis_end_date = self.current_price_date
self.recommendation_set = None
self.raw_dataframe = None
self.recommendation_dataframe = None
def mark_to_market(data_frame: object, price_date: datetime):
"""
Peforms a Mark to Market on a Pandas dataframe representing
a ranked portfolio, and given a price date. This is used
to calculate returns on a portfolio generated by one of the
strategies
The dataframe must contain the following columuns:
* ticker
* analysis_price
and will add:
* current_price
* actual_return
Parmeters
---------
data_frame : Pandas DataFrame
portfolio dataframe
price_date : datetime
price date, current or historical
Returns
---------
A new dataframe with the added columns
Raises
---------
ValidationError if parameters are incorrect
DataError if there are problems reading price data
"""
if (data_frame is None or price_date is None):
raise ValidationError(
"Invalid Parameters supplied to Mark to Market calculation", None)
if ('ticker' not in data_frame.columns
or 'analysis_price' not in data_frame.columns):
raise ValidationError(
"Could not extract required fields for Mark to Market calculation",
None)
mmt_prices = []
for ticker in data_frame['ticker']:
try:
latest_price = intrinio_data.get_latest_close_price(
ticker, price_date, 5)[1]
mmt_prices.append(latest_price)
except Exception as e:
raise DataError("Could not perform MMT calculation", e)
data_frame['current_price'] = mmt_prices
data_frame['actual_return'] = (
data_frame['current_price'] -
data_frame['analysis_price']) / data_frame['analysis_price']
return data_frame
def mark_to_market(data_frame: object, ticker_col_name: str,
price_col_name: str, price_date: date):
"""
Peforms a Mark to Market on a Pandas dataframe representing
a set of stocks given a price date. This is used
to calculate returns on a portfolio generated by one of the
strategies
The dataframe must contain a ticker and price column, which are defined
via the parameters and will add:
* current_price
* actual_return
Parmeters
---------
data_frame: Pandas DataFrame
Portfolio dataframe
ticker_col_name: str
Name of the ticker column
price_col_name: str
Name of the price column
price_date: date
Price date, current or historical
Returns
---------
A new dataframe with the added columns
"""
if (data_frame is None or price_date is None):
raise ValidationError(
"Invalid Parameters supplied to Mark to Market calculation", None)
if (ticker_col_name not in data_frame.columns
or price_col_name not in data_frame.columns):
raise ValidationError(
"Could not extract required fields for Mark to Market calculation",
None)
mmt_prices = []
for ticker in data_frame[ticker_col_name]:
try:
latest_price = intrinio_data.get_daily_stock_close_prices(
ticker, price_date, price_date)
mmt_prices.append(latest_price[price_date.strftime("%Y-%m-%d")])
except Exception as e:
raise DataError("Could not perform MMT calculation", e)
data_frame['current_price'] = mmt_prices
data_frame['actual_return'] = (
data_frame['current_price'] -
data_frame[price_col_name]) / data_frame[price_col_name]
return data_frame
def test_simple_exception_nocause(self):
self.assertEqual(str(ValidationError("Cannot do XYZ", None)), "Validation Error: Cannot do XYZ")
self.assertEqual(str(CalculationError("Cannot do XYZ", None)), "Calculation Error: Cannot do XYZ")
self.assertEqual(str(DataError("Cannot do XYZ", None)), "Data Error: Cannot do XYZ")
self.assertEqual(str(ReportError("Cannot do XYZ", None)), "Report Error: Cannot do XYZ")
self.assertEqual(repr(ValidationError("Cannot do XYZ", None)), "Validation Error: Cannot do XYZ")
self.assertEqual(repr(CalculationError("Cannot do XYZ", None)), "Calculation Error: Cannot do XYZ")
self.assertEqual(repr(DataError("Cannot do XYZ", None)), "Data Error: Cannot do XYZ")
self.assertEqual(repr(ReportError("Cannot do XYZ", None)), "Report Error: Cannot do XYZ")
def generate_report(self, report_filename: str):
"""
Generates a report and all associated worksheets
Parameters
----------
report_filename : str
Name of the output report
Raises
----------
ValidationError
In case the report could not be generate because it was not properly
initialized
ReportError
In case of any errors preparing or generating the report.
Returns
-------
None
"""
wb = Workbook()
wb.remove(wb.active)
#
# Assemble the report
#
if len(self.worksheet_list) == 0:
raise ValidationError("No worksheets were supplied to the report",
None)
if report_filename is None or report_filename == "":
raise ValidationError("No report filename was supplied", None)
for (report_worksheet, worksheet_tile,
dcf_model) in self.worksheet_list:
self.price_dict[worksheet_tile] = dcf_model.calculate_dcf_price()
source_worksheet = report_worksheet.create_worksheet(
dcf_model.get_itermediate_results())
target_worksheet = wb.create_sheet(worksheet_tile)
self.__copy_worksheet__(source_worksheet, target_worksheet)
output_report_name = '%s%s' % (self.output_path, report_filename)
try:
wb.save(output_report_name)
except Exception as e:
raise ReportError("Error saving report", e)
def validate_commandline_parameters(year: int, month: int,
current_price_date: datetime):
'''
Validates command line parameters and throws an exception
if they are not properly set.
'''
if (year < 2000 or (month not in range(1, 13))):
raise ValidationError("Parameters out of range", None)
if datetime(year, month, 1) >= current_price_date:
raise ValidationError(
"Price Date must be in future compared to analysis period", None)
def __init__(self, ticker_list: list, data_year: int, data_month: int,
output_size: int):
"""
Initializes the class with the ticker list, a year and a month.
The year and month are used to set the context of the analysis,
meaning that financial data will be used for that year/month.
This is done to allow the analysis to be run in the past and test the
quality of the results.
Parameters
------------
ticker_list : list of tickers to be included in the analisys
ticker_source_name : The source of the ticker list. E.g. DOW30, or SP500
year : analysis year
month : analysis month
output_size : number of recommended securities that will be returned
by this strategy
"""
if (ticker_list is None or len(ticker_list) == 0):
raise ValidationError("No ticker list was supplied", None)
if len(ticker_list) < 2:
raise ValidationError("You must supply at least 2 ticker symbols",
None)
if output_size <= 0:
raise ValidationError("Output size must be at least 1", None)
(self.analysis_start_date,
self.analysis_end_date) = intrinio_util.get_month_date_range(
data_year, data_month)
if (self.analysis_end_date > datetime.now()):
logging.debug("Setting analysis end date to 'today'")
self.analysis_end_date = datetime.now()
self.ticker_list = ticker_list
self.output_size = output_size
self.data_date = "%d-%d" % (data_year, data_month)
self.recommendation_set = None
self.raw_dataframe = None
self.recommendation_dataframe = None
def from_configuration(cls, configuration: object, app_ns: str):
'''
See BaseStrategy.from_configuration for documentation
'''
today = pd.to_datetime('today').date()
try:
config_params = dict(configuration.config[cls.CONFIG_SECTION])
ticker_file_name = config_params['ticker_list_file_name']
output_size = int(config_params['output_size'])
except Exception as e:
raise ValidationError(
"Could not read MACD Crossover Strategy configuration parameters",
e)
ticker_list = TickerList.try_from_s3(app_ns, ticker_file_name)
analysis_period = (pd.Period(today, 'M') - 1).strftime("%Y-%m")
current_price_date = util.get_business_date(
constants.BUSINESS_DATE_DAYS_LOOKBACK,
constants.BUSINESS_DATE_CUTOVER_TIME)
return cls(ticker_list, analysis_period, current_price_date,
output_size)
def __init__(self, path, **kwargs):
'''
Initializes the cache
Parameters
----------
path : str
The path where the cache will be located
max_cache_size_bytes : int (kwargs)
(optional) the maximum size of the cache in bytes
Returns
-----------
A tuple of strings containing the start and end date of the fiscal period
'''
try:
max_cache_size_bytes = kwargs['max_cache_size_bytes']
except KeyError:
# default max cache is 4GB
max_cache_size_bytes = 4e9
util.create_dir(path)
try:
self.disk_cache = Cache(path, size_limit=int(max_cache_size_bytes))
except Exception as e:
raise ValidationError('invalid max cache size', e)
log.debug("Cache was initialized: %s" % path)
def get_service_inputs(app_ns: str):
'''
Returns the required inputs for the recommendation service given
the application namespace used to identify the appropriate cloud resources.
Returns
----------
A tuple containing the latest SecurityRecommendationSet
and Portfolio objects. If a portfolio does not exist, it will
create a new one.
'''
log.info("Loading latest recommendations")
recommendation_set = SecurityRecommendationSet.from_s3(app_ns)
if not recommendation_set.is_current(datetime.now()):
raise ValidationError("Current recommendation set is not valid", None)
try:
log.info("Loading current portfolio")
pfolio = Portfolio.from_s3(app_ns)
except AWSError as e:
if e.resource_not_found():
pfolio = None
else:
raise e
return (pfolio, recommendation_set)
def _read_company_data_point(ticker: str, tag: str):
"""
Helper function that will read the Intrinio company API for the supplied ticker
and return the value
Returns
-------
The numerical value of the datapoint
"""
# check the cache first
cache_key = "%s-%s-%s-%s" % (INTRINIO_CACHE_PREFIX,
"company_data_point_number", ticker, tag)
api_response = cache.read(cache_key)
if api_response is None:
# else call the API directly
try:
api_response = COMPANY_API.get_company_data_point_number(
ticker, tag)
cache.write(cache_key, api_response)
except ApiException as ae:
raise DataError(
"Error retrieving ('%s') -> '%s' from Intrinio Company API" %
(ticker, tag), ae)
except Exception as e:
raise ValidationError(
"Error parsing ('%s') -> '%s' from Intrinio Company API" %
(ticker, tag), e)
return api_response
def get_business_date_offset(business_date: date, days_offset: int):
'''
Returns the business date offest by 'days_offset' business date.
For example, the day before the observed 4th of July will return
the following Monday:
(2020/07/02, 1) -> 2020/07/06
if the business date is not valid, the method will throw a ValidationError
'''
nyse_cal = mcal.get_calendar('NYSE')
if days_offset > 0:
market_calendar = nyse_cal.schedule(
business_date,
business_date + timedelta(days=int(days_offset * 1.5)))
else:
market_calendar = nyse_cal.schedule(
business_date + timedelta(days=int(days_offset * 1.5)),
business_date)
# reverse the calendar order
market_calendar = market_calendar.iloc[::-1]
# if business_date is not valid, raise an exception
try:
market_calendar.index.get_loc(business_date)
except Exception as e:
raise ValidationError(
"Cannot offset %s by %d days because %s is not a valid business date"
% (business_date, days_offset, business_date), e)
return market_calendar.index[abs(days_offset)].to_pydatetime().date()
def test_simple_exception_with_stringcause(self):
validation_error = ValidationError("Cannot do XYZ", "Some Error")
self.assertEqual(
str(validation_error),
"Validation Error: Cannot do XYZ. Caused by: Some Error")
def _read_price_metrics(self, ticker_symbol: str):
'''
Helper function that downloads the data required by the strategy.
Returns
-------
A Tuple with the following elements:
current_price: float
The Current price for the ticker symbol
macd_lines: list
The past 3 days of MACD values
signal_lines
The past 3 days of MACD Singal values
'''
dict_key = self.analysis_date.strftime("%Y-%m-%d")
current_price_dict = intrinio_data.get_daily_stock_close_prices(
ticker_symbol, self.analysis_date, self.analysis_date
)
macd_dict = intrinio_data.get_macd_indicator(
ticker_symbol, self.analysis_date, self.analysis_date, self.macd_fast_period, self.macd_slow_period, self.macd_signal_period
)
try:
current_price = current_price_dict[dict_key]
macd_line = macd_dict[dict_key]['macd_line']
signal_line = macd_dict[dict_key]['signal_line']
except Exception as e:
raise ValidationError(
"Could not read pricing data for %s" % ticker_symbol, e)
return (current_price, macd_line, signal_line)
def from_configuration(cls, configuration: object, app_ns: str):
'''
See BaseStrategy.from_configuration for documentation
'''
analysis_date = util.get_business_date(
constants.BUSINESS_DATE_DAYS_LOOKBACK, constants.BUSINESS_DATE_CUTOVER_TIME)
try:
config_params = dict(configuration.config[cls.CONFIG_SECTION])
ticker_file_name = config_params['ticker_list_file_name']
divergence_factor_threshold = float(
config_params['divergence_factor_threshold'])
macd_fast_period = int(config_params['macd_fast_period'])
macd_slow_period = int(config_params['macd_slow_period'])
macd_signal_period = int(config_params['macd_signal_period'])
except Exception as e:
raise ValidationError(
"Could not read MACD Crossover Strategy configuration parameters", e)
finally:
configuration.close()
ticker_list = TickerList.try_from_s3(app_ns, ticker_file_name)
return cls(ticker_list, analysis_date, divergence_factor_threshold, macd_fast_period, macd_slow_period, macd_signal_period)
def get_credentials():
'''
Read the TD Credentials from the environment or throws an exception
'''
def read_from_env(env_name: str):
'''
reads a TD credential variable from the environment.
If not found add it to the "missing" list to improve
the error message.
'''
try:
return os.environ.get(env_name)
except Exception:
missing_variables.append(env_name)
missing_variables.clear()
td_account_id = read_from_env('TDAMERITRADE_ACCOUNT_ID')
td_client_id = read_from_env('TDAMERITRADE_CLIENT_ID')
td_refresh_token = read_from_env('TDAMERITRADE_REFRESH_TOKEN')
if len(missing_variables) > 0:
raise ValidationError(
"Could not read TDAmeritrade credentials from environment. Missing %s"
% str(missing_variables), None)
return (td_account_id, td_client_id, td_refresh_token)
def reprice(self, price_date: datetime):
'''
Reads the current prices, computes the latest returns
and updates the portfolio object.
'''
# Update the current returns in the securities set
for security in self.model['securities_set']:
analysis_price = security['analysis_price']
(price_date_str,
latest_price) = intrinio_data.get_latest_close_price(
security['ticker_symbol'], price_date, 5)
security['current_price'] = latest_price
# if a portfolio exsts, reprice it too
if not self.is_empty():
for security in self.model['current_portfolio']['securities']:
purchase_price = security['purchase_price']
(price_date_str,
latest_price) = intrinio_data.get_latest_close_price(
security['ticker_symbol'], price_date, 5)
security['current_price'] = latest_price
self.recalc_returns()
# finally set the price date
try:
price_date = parser.parse(price_date_str)
except Exception as e:
raise ValidationError(
"Could parse price date returned by Intrinio API", e)
self.model['price_date'] = util.date_to_iso_utc_string(price_date)
log.info("Repriced portfolio for date of %s" % str(price_date))
def get_year_date_range(year: int, extend_by_days: int):
"""
returns the first and last day of the supplied year formatted in a way
that can be supplied to the Intrinion SDK
For example:
2018 -> ('2018-01-01', '2019-12-31')
the end date can be extended by "extend_by_days"
For example:
(2018, 10) -> ('2018-01-01', '2019-01-10')
Parameters
----------
year : int
the year in question
extend_by_days : int
number of days by which to extend the end date period.
Returns
-----------
A tuple of strings containing the start and end date.
"""
__validate_year__(year)
if extend_by_days < 0 or extend_by_days > 350:
raise ValidationError(
"Invalid extend_by_days. Must between 0 and 350", None)
start = datetime(year, 1, 1)
end = datetime(year, 12, 31) + timedelta(days=extend_by_days)
return(date_to_string(start), date_to_string(end))
def get_service_inputs(app_ns: str):
'''
Returns the required inputs for the recommendation service given
the application namespace used to identify the appropriate cloud resources.
Returns
----------
A tuple containing the latest SecurityRecommendationSet
and Portfolio objects. If a portfolio does not exist, it will
create a new one.
'''
log.info("Loading latest recommendations")
recommendation_set = SecurityRecommendationSet.from_s3(
app_ns, constants.S3_PRICE_DISPERSION_RECOMMENDATION_SET_OBJECT_NAME)
business_date = util.get_business_date(
constants.BUSINESS_DATE_DAYS_LOOKBACK,
constants.BUSINESS_DATE_CUTOVER_TIME)
if not recommendation_set.is_current(business_date):
raise ValidationError("Current recommendation set is not valid", None)
try:
log.info("Loading current portfolio")
pfolio = Portfolio.from_s3(app_ns, constants.S3_PORTFOLIO_OBJECT_NAME)
except AWSError as e:
if e.resource_not_found():
pfolio = None
else:
raise e
return (pfolio, recommendation_set)
def __init__(self, template_name: str):
super().__init__()
# assign this to a local variable so that it can be mocked
if template_name == None or len(template_name) == 0:
raise ValidationError("Invalid parameters", None)
self.template_name = template_name
def test_retry_server_errors_validtion_error(self):
mock = Mock(side_effect=ValidationError("Mock Error", None))
test_function = intrinio_data.retry_server_errors(mock)
with self.assertRaises(ValidationError):
test_function()
self.assertEqual(mock.call_count, 1)
def from_parameters(cls, creation_date: datetime, valid_from: datetime,
valid_to: datetime, price_date: datetime,
strategy_name: str, security_type: str,
securities_set: dict):
'''
Initializes This class by supplying all required parameters.
The "securities_set" is a ticker->price dictionary. E.g.
{
'AAPL': 123.45,
'XO' : 234.56,
...
}
'''
if (strategy_name is None or strategy_name == ""
or security_type is None or security_type == ""
or securities_set is None or len(securities_set) == 0):
raise ValidationError(
"Could not initialize Portfolio objects from parameters", None)
try:
cls.model = {
"set_id": str(uuid.uuid1()),
"creation_date": util.date_to_iso_utc_string(creation_date),
"valid_from": util.date_to_iso_string(valid_from),
"valid_to": util.date_to_iso_string(valid_to),
"price_date": util.date_to_iso_string(price_date),
"strategy_name": strategy_name,
"security_type": security_type,
"securities_set": []
}
for ticker in securities_set.keys():
cls.model['securities_set'].append({
"ticker_symbol":
ticker,
"price":
securities_set[ticker]
})
except Exception as e:
raise ValidationError(
"Could not initialize Portfolio objects from parameters", e)
return cls.from_dict(cls.model)
def test_simple_exception_with_chainedcause(self):
root_cause = Exception("Root Cause")
chained_cause = Exception("Some reason", root_cause)
ve = ValidationError("Cannot do XYZ", chained_cause)
# mac os and linux produce slightly different results
self.assertTrue("Validation Error: Cannot do XYZ. Caused by: ('Some reason', Exception('Root Cause" in str(ve))
def test_simple_exception_with_exceptioncause(self):
empty_dict = {}
try:
empty_dict['XXX']
self.fail("Error in test setup")
except KeyError as ke:
ve = ValidationError("Cannot do XYZ", ke)
self.assertEqual(str(ve), "Validation Error: Cannot do XYZ. Caused by: 'XXX'")
def validate_model(self):
'''
(Re)validates the model
'''
try:
validate(self.model, self.schema,
format_checker=jsonschema.FormatChecker())
except Exception as e:
raise ValidationError(
"Could not validate %s model" % self.model_name, e)
def get_stackname_from_stackarn(arn: str):
# arn:aws:cloudformation:region:acct:stack/app-infra-base/c9481160-6df5-11ea-ac9f-121b58656156
try:
arn_elements = arn.split(':')
stack_id = arn_elements[5]
stack_elements = stack_id.split("/")
return stack_elements[1]
except Exception as e:
raise ValidationError("Could not parse stack ID from arn", e)
def datetime_to_iso_utc_string(date: datetime):
'''
Converts a date object into an 8601 string using UTC
'''
if date is None:
return "None"
try:
return date.astimezone(pytz.UTC).isoformat()
except Exception as e:
raise ValidationError("Could not convert date to string", e)
def get_business_date_list(start_date: date, end_date: date):
'''
given a calendar date, returns the nearest past business date
'''
if start_date >= end_date:
raise ValidationError("Start date must be before end date")
nyse_cal = mcal.get_calendar('NYSE')
market_calendar = nyse_cal.schedule(start_date, end_date)
business_date_list = [date.date() for date in list(market_calendar.index)]
return business_date_list