def _parse_json(json_text, fin):
"""
Parse the json response from the morningstar
:param json_text: the json response
:type json_text: str
:param fin: the morningstar financial object of the stock
:type fin: financial.Financial
:return: whether success or not
"""
stock = fin.stock
try:
json_obj = json.loads(json_text)
except ValueError:
raise fetcher_exception.FetcherException(
"Decoding JSON has failed for " + stock)
if json_obj is None:
raise fetcher_exception.FetcherException(
"Decoding JSON has failed for " + stock)
# Parse the PriceDataList
if not HistoricalFetcher._parse_price_data(json_obj, fin):
return False
# Parse the DividendData
if not HistoricalFetcher._parse_dividend_data(json_obj, fin):
return False
return True
def _parse_financial_with_dates(lines,
dates,
line_prefix,
has_currency=True,
has_unit=True,
required_unit="mil"):
values = {}
line_idx = FundamentalFetcher._find_line_startwith(lines, line_prefix)
# Get the currency and unit
if line_idx == -1:
raise fetcher_exception.FetcherException(
"Cannot find line starting with" + line_prefix)
line = lines[line_idx]
line_postfix = line[len(
line_prefix):] # the line by cutting the prefix part
currency, unit = FundamentalFetcher._get_currency_and_unit(
line_postfix, has_currency, has_unit)
tokens = FundamentalFetcher._parse_csv_line(line)
for i in range(1, len(tokens)):
token = tokens[i].replace(',', '').strip()
if len(token) == 0:
continue
value = float(token)
if has_unit:
value = FundamentalFetcher._unit_convert(
unit, value, required_unit)
date = dates[i - 1]
values[date] = value
if not has_currency:
return values
else:
return values, currency
def _parse_dividend_data(json_obj, fin):
"""
Extract the Dividend data from the parsed JSON object into the financial object
:param json_obj: the parsed json object
:type json_obj: dict
:param fin: the morningstar financial object of the stock
:type fin: financial.Financial
:return: whether success or not
"""
stock = fin.stock
if HistoricalFetcher.DIVIDEND_DATA not in json_obj:
raise fetcher_exception.FetcherException(
HistoricalFetcher.DIVIDEND_DATA +
" is not in the json response for " + stock)
dividend_data = json_obj[HistoricalFetcher.DIVIDEND_DATA]
dividends = {}
num_points = len(dividend_data)
if num_points == 0:
LogInfo.info(stock + " has no dividend.")
else:
for idx in range(num_points):
record = dividend_data[idx]
dividend_date = record["Date"]
dividend_type = record["Type"]
if dividend_type == "Dividend":
desc = record["Desc"] # e.g., "Dividends:0.5200"
tokens = desc.split(":")
div_str = tokens[1].strip().replace("<br>", "")
dividends[dividend_date] = float(div_str) # e.g., 0.5200
fin.stock_dividend_date = dividends
return True
def _unit_convert(src_unit, src_value, dest_unit):
if src_unit == dest_unit:
return src_value
if src_unit == "" and dest_unit == "mil":
return src_value / (1000.0 * 1000.0)
if src_unit == "mil" and dest_unit == FundamentalFetcher.NO_UNIT:
return src_value * (1000.0 * 1000.0)
raise fetcher_exception.FetcherException(
"Does not support to convert " + str(src_value) + " into " +
dest_unit)
def fetch(self, fin, start_date, end_date, num_retries=3, use_cache=False):
"""
Fetch the fundamental data of a security
:param fin: the morningstar financial object of the stock
:type fin: financial.Financial
:param start_date: the start date
:type start_date: str
:param end_date: the end date
:type end_date: str
:param num_retries: the number of retries
:type num_retries: int
:param use_cache: whether to use cache if cache exists
:type use_cache: bool
:return: whether success or not
"""
stock = fin.stock
url = (
r'http://globalquote.morningstar.com/globalcomponent/RealtimeHistoricalStockData.ashx?'
r'ticker={0}&showVol=true&dtype=his&f=d&curry=USD'
r'&range={1}|{2}&isD=true&isS=true&hasF=true&ProdCode=DIRECT'.
format(stock, start_date, end_date))
for try_idx in range(num_retries):
try:
filename = "/tmp/" + stock + "_prices.json"
if use_cache and HistoricalFetcher._has_cache(filename):
tmp_file = open(filename, "r")
json_text = tmp_file.read()
tmp_file.close()
else:
response = urllib2.urlopen(url)
json_text = response.read()
if len(json_text.strip()) == 0 or json_text.strip(
) == 'null':
raise fetcher_exception.FetcherException(
"Empty response of the http request for " + stock)
tmp_file = open(filename, "w")
tmp_file.write(json_text)
tmp_file.close()
success = HistoricalFetcher._parse_json(json_text, fin)
if success:
return True
except Exception as err:
traceback.print_exc()
LogInfo.info(stock + " : " + err.message + " in the " +
str((try_idx + 1)) + " time for " + stock)
if try_idx == num_retries - 1:
LogInfo.error('Failed to retrieve information for ' +
stock)
return False
def _get_currency_and_unit(line, has_currency=True, has_unit=True):
if (not has_currency) and (not has_unit):
return FundamentalFetcher.NO_CURRENCY, FundamentalFetcher.NO_UNIT
line = line.replace("*", " ")
line = line.strip()
tokens_by_comma = line.split(",")
if len(tokens_by_comma) == 0:
raise fetcher_exception.FetcherException("The line is empty : " +
line)
tokens = tokens_by_comma[0].split()
tokens = [val.strip().lower() for val in tokens]
if len(tokens) == 0:
raise fetcher_exception.FetcherException(
"The line has no currency or unit : " + line)
elif len(tokens) == 1:
if has_currency and (not has_unit):
return tokens[-1], FundamentalFetcher.NO_UNIT
elif (not has_currency) and has_unit:
return FundamentalFetcher.NO_CURRENCY, tokens[-1]
else:
raise fetcher_exception.FetcherException(
"The line only has currency or unit : " + line)
else:
return tokens[-2], tokens[-1]
def _parse_financial(self, lines, fin):
line_idx = self._find_line_index(lines, self.FINANCIAL_SECTION_HEADER)
if line_idx == -1:
raise fetcher_exception.FetcherException(
"The html contains no " + self.FINANCIAL_SECTION_HEADER)
# Get the section of lines
sec_lines = FundamentalFetcher._get_section_lines(lines, line_idx)
line_dates = lines[line_idx + 1]
# Get the dates
dates = FundamentalFetcher._get_dates(line_dates)
# Get the financial values
fin.revenue_mil, fin.revenue_currency = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.REVENUE_LINE_PREFIX,
required_unit="mil")
fin.net_income_mil, fin.net_income_currency = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.NET_INCOME_LINE_PREFIX,
required_unit="mil")
fin.free_cash_flow_mil, fin.free_cash_flow_currency = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.FREE_CASH_FLOW_LINE_PREFIX,
required_unit="mil")
fin.book_value_per_share, fin.book_value_currency = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.BOOK_VALUE_PER_SHARE_PREFIX,
has_currency=True, has_unit=False)
fin.share_mil = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.SHARE_MIL_PREFIX,
has_currency=False, has_unit=False)
fin.operating_income_mil, fin.operating_income_currency = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.OPERATING_INCOME_PREFIX,
required_unit="mil")
gross_margin = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.GROSS_MARGIN,
has_currency=False, has_unit=False)
# change the percentage number into real numbers
fin.gross_margin = dict(
(date, value / 100.0) for date, value in gross_margin.items())
fin.dividends, fin.dividend_currency = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.DIVIDENDS,
has_currency=True, has_unit=False)
def _parse_liquidity(self, lines, fin):
line_idx = self._find_line_index(lines, self.LIQUIDITY_SECTION_HEADER)
if line_idx == -1:
raise fetcher_exception.FetcherException(
"The html contains no " + self.FINANCIAL_HEALTH_SECTION_HEADER)
# Get the section of lines
sec_lines = FundamentalFetcher._get_section_lines(lines, line_idx)
line_dates = lines[line_idx]
# Get the dates
dates = FundamentalFetcher._get_dates(line_dates)
# Get the financial values
fin.current_ratio = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.CURRENT_RATIO_PREFIX,
has_currency=False, has_unit=False)
fin.debt_to_equity = \
FundamentalFetcher._parse_financial_with_dates(sec_lines, dates, self.DEBT_EQUITY_PREFIX,
has_currency=False, has_unit=False)
def fetch(self, fin, num_retries=3, use_cache=False):
"""
Fetch the fundamental data of a security
:param fin: the morningstar financial object of the stock
:type fin: financial.Financial
:param num_retries: the number of retries
:type num_retries: int
:param use_cache: whether to use cache if cache exists
:type use_cache: bool
:return: whether success or not
"""
stock = fin.stock
url = (
r'http://financials.morningstar.com/ajax/exportKR2CSV.html?' +
r'&callback=?&t={0}®ion=usa&culture=en-US&cur=USD'.format(stock)
)
for try_idx in range(num_retries):
try:
filename = "/tmp/" + stock + ".csv"
if use_cache and FundamentalFetcher._has_cache(filename):
tmp_file = open(filename, "r")
html = tmp_file.read()
tmp_file.close()
else:
response = urllib2.urlopen(url)
html = response.read()
if len(html.strip()) == 0:
raise fetcher_exception.FetcherException(
"Empty response of the http request.")
tmp_file = open(filename, "w")
tmp_file.write(html)
tmp_file.close()
self._parse_html(html, fin)
return True
except Exception as err:
traceback.print_exc()
LogInfo.info(stock + " : " + err.message + " in the " +
str((try_idx + 1)) + " time")
if try_idx == num_retries - 1:
LogInfo.error('Failed to retrieve information for ' +
stock)
return False
def _parse_price_data(json_obj, fin):
"""
Extract the Price data from the parsed JSON object into the financial object
:param json_obj: the parsed json object
:type json_obj: dict
:param fin: the morningstar financial object of the stock
:type fin: financial.Financial
:return: whether success or not
"""
stock = fin.stock
if HistoricalFetcher.PRICE_DATA_LIST not in json_obj:
raise fetcher_exception.FetcherException(
HistoricalFetcher.PRICE_DATA_LIST +
" is not in the json response for " + stock)
if HistoricalFetcher.VOLUME_LIST not in json_obj:
raise fetcher_exception.FetcherException(
HistoricalFetcher.VOLUME_LIST +
" is not in the json response for " + stock)
price_data_list = json_obj[HistoricalFetcher.PRICE_DATA_LIST]
if len(price_data_list) == 0:
raise fetcher_exception.FetcherException(
HistoricalFetcher.PRICE_DATA_LIST + " is empty for " + stock)
price_data = price_data_list[0]
if HistoricalFetcher.DATA_POINTS not in price_data:
raise fetcher_exception.FetcherException(
HistoricalFetcher.DATA_POINTS + " is not in the " +
HistoricalFetcher.PRICE_DATA_LIST + " for " + stock)
price_data_points = price_data[HistoricalFetcher.DATA_POINTS]
if HistoricalFetcher.DATE_INDEXS not in price_data:
raise fetcher_exception.FetcherException(
HistoricalFetcher.DATE_INDEXS + " is not in the " +
HistoricalFetcher.PRICE_DATA_LIST + "for " + stock)
date_indexes = price_data[HistoricalFetcher.DATE_INDEXS]
num_points = len(price_data_points)
volume_data_list = json_obj[HistoricalFetcher.VOLUME_LIST]
if HistoricalFetcher.DATA_POINTS not in volume_data_list:
raise fetcher_exception.FetcherException(
HistoricalFetcher.DATA_POINTS + " is not in the " +
HistoricalFetcher.VOLUME_LIST + " for " + stock)
volume_data_points = volume_data_list[HistoricalFetcher.DATA_POINTS]
assert num_points == len(date_indexes)
assert num_points == len(volume_data_points)
close_prices = {}
highest_prices = {}
lowest_prices = {}
open_prices = {}
volumes = {}
for idx in range(num_points):
date_index = date_indexes[idx]
date_str = HistoricalFetcher._convert_to_date(date_index)
price_point = price_data_points[idx]
assert len(price_point) == 4
[close_price, highest_price, lowest_price,
open_price] = price_point
close_prices[date_str] = close_price
highest_prices[date_str] = highest_price
lowest_prices[date_str] = lowest_price
open_prices[date_str] = open_price
volumes[date_str] = float(volume_data_points[idx]) * 1000 * 1000
fin.stock_daily_close_price = close_prices
fin.stock_daily_highest_price = highest_prices
fin.stock_daily_lowest_price = lowest_prices
fin.stock_daily_open_price = open_prices
fin.stock_daily_price_currency = 'USD'
fin.stock_daily_volume = volumes
return True