def test_stock_in_which_index_multiple(self):
soup = scrap.get_soup_code_from_file("data/bo_sap-aktie.html")
link_items = parse.get_listed_indizes(soup)
asserted_idx_values = [
"TecDAX",
"DAX",
"STOXX 50",
"EURO STOXX 50",
"S&P 400 MidCap",
"EURO STOXX Technology",
"Prime All Share",
"LDAX",
"LTecDAX",
"HDAX",
"DivDAX",
"NYSE International 100",
"CDAX",
"EURO STOXX",
"TecDAX Kursindex",
"DAX Kursindex",
"BX Swiss - EMEA",
"XDAX",
"DAXglobal Sarasin Sustainability Germany Index EUR",
"L&S DAX Indikation",
"QIX Deutschland",
"DAXglobal Sarasin Sustainability Germany",
"Schatten-Index-TecDAX",
]
self.assertEqual(asserted_idx_values, link_items)
soup2 = scrap.get_soup_code_from_file("data/bo_3i-Aktie_overview.html")
link_items2 = parse.get_listed_indizes(soup2)
asserted_indizes_values2 = ["FTSE 100", "FTSE Allshare"]
self.assertEqual(asserted_indizes_values2, link_items2)
def test_get_market_cap(self):
soup = scrap.get_soup_code_from_file("data/bo_tesla-aktie.html")
market_cap_value = parse.get_market_cap(soup)
asserted_market_cap = 37530
self.assertEqual(asserted_market_cap, market_cap_value)
soup2 = scrap.get_soup_code_from_file("data/bo_3i-Aktie_overview.html")
market_cap_value2 = parse.get_market_cap(soup2)
asserted_market_cap2 = 9600
self.assertEqual(asserted_market_cap2, market_cap_value2)
soup3 = scrap.get_soup_code_from_file(
"data/bo_london_stock_exchange-Aktie_overview.html")
market_cap_value3 = parse.get_market_cap(soup3)
asserted_market_cap3 = 15890
self.assertEqual(asserted_market_cap3, market_cap_value3)
def write_stock_last_quarterly_figures_date_to_db():
stock_list = db.get_list(table=cst.TABLE_STOCKS, columns=cst.COLUMN_URI)
file_list = [
cst.PATH_STOCK_DATES + stock[:-6] + cst.HTML_EXTENSION
for stock in stock_list
]
for file in file_list:
stock_uri = file.split("/")[-1][:-5] + "-Aktie"
stock_targets_soup = scrap.get_soup_code_from_file(file)
if stock_targets_soup is None:
logger.warning(
"Write Stock Quaterlyfigures Data to DB: Soup is None for %s" %
str(file))
continue
try:
last_figures_date = parse.get_last_quarterly_figures_date(
stock_targets_soup)
if last_figures_date is None:
continue
else:
# db.write_single_stock_dates_data_to_db(stock_uri, last_figures_date)
db.upsert_item(
table=cst.TABLE_STOCK_DATES,
primary_keys=[cst.COLUMN_STOCK_URI, cst.COLUMN_DATE],
current_date=last_figures_date,
quarterly="Quarterly/Yearly",
)
except:
logger.exception(
"Write Stock Quaterly Data to DB: Exception for stock: %s" %
stock_uri)
def write_stock_targets_data_to_db():
stock_list = db.get_list(table=cst.TABLE_STOCKS, columns=cst.COLUMN_URI)
file_list = [
cst.PATH_STOCK_TARGETS + stock[:-6] + cst.HTML_EXTENSION
for stock in stock_list
]
for file in file_list:
stock_uri = file.split("/")[-1][:-5] + "-Aktie"
stock_targets_soup = scrap.get_soup_code_from_file(file)
if stock_targets_soup is None:
logger.warning(
"Write Stock Targets Data to DB: Soup is None for %s" %
str(file))
continue
try:
buy, hold, sell = parse.get_analyst_ratings(stock_targets_soup)
db.upsert_item(
table=cst.TABLE_COMPANY_DATA,
primary_keys=[cst.COLUMN_STOCK_URI, cst.COLUMN_DATE],
current_date=date.get_current_date(),
stock_uri=stock_uri,
analyst_buy=buy,
analyst_hold=hold,
analyst_sell=sell,
)
except:
logger.exception(
"Write Stock Targets Data to DB: Exception for stock: %s" %
stock_uri)
def write_stock_estimates_data_to_db():
stock_list = db.get_list(table=cst.TABLE_STOCKS, columns=cst.COLUMN_URI)
file_list = [
cst.PATH_STOCK_ESTIMATES + stock[:-6] + cst.HTML_EXTENSION
for stock in stock_list
]
for file in file_list:
stock_uri = file.split("/")[-1][:-5] + "-Aktie"
stock_estimate_soup = scrap.get_soup_code_from_file(file)
if stock_estimate_soup is None:
logger.warning(
"Write Stock Estimate Data to DB: Soup is None for %s" %
str(file))
continue
try:
eps_0, eps_p1 = parse.get_result_per_share_current_and_next_year(
stock_estimate_soup)
db.upsert_item(
table=cst.TABLE_COMPANY_DATA,
primary_keys=[cst.COLUMN_STOCK_URI, cst.COLUMN_DATE],
current_date=date.get_current_date(),
stock_uri=stock_uri,
eps_0=eps_0,
eps_p1=eps_p1)
except:
logger.exception(
"Write Stock Estimate Data to DB: Exception for stock: %s" %
stock_uri)
def test_get_result_per_share_current_and_next_year(self):
soup = scrap.get_soup_code_from_file("data/bo_schaetzungen.html")
result_2018, result_2019 = parse.get_result_per_share_current_and_next_year(
soup)
asserted_result_2018 = 8.22
asserted_result_2019 = 9.54
self.assertEqual(asserted_result_2018, result_2018)
self.assertEqual(asserted_result_2019, result_2019)
def test_get_bygone_dates(self):
soup = scrap.get_soup_code_from_file("data/bo_termine.html")
result = parse.get_bygone_dates(soup)
asserted_result = [
["adidas AG", "2018-08-09", "Q2 2018 Earnings Release", "Past"],
["adidas AG", "2018-05-09", "Hauptversammlung", "Past"],
["adidas AG", "2017-11-09", "Q3 2017", "Past"],
]
self.assertEqual(asserted_result, result)
def test_get_analyst_ratings(self):
soup = scrap.get_soup_code_from_file("data/bo_kursziele.html")
no_buy, no_hold, no_sell = parse.get_analyst_ratings(soup)
asserted_no_buy = 15
asserted_no_hold = 8
asserted_no_sell = 0
self.assertEqual(asserted_no_buy, no_buy)
self.assertEqual(asserted_no_hold, no_hold)
self.assertEqual(asserted_no_sell, no_sell)
def test_get_historic_prices(self):
soup = scrap.get_soup_code_from_file("data/bo_index_history_big.html")
table_list = parse.get_historic_prices_from_history(soup)
asserted_list = [
["01.10.2018", "9.127,05", "9.094,28", "9.155,65", "9.084,22"],
["02.10.2018", "9.087,32", "9.076,57", "9.090,46", "9.050,50"],
["03.10.2018", "9.175,21", "9.126,31", "9.194,99", "9.123,48"],
]
self.assertEqual(asserted_list, table_list[:3])
def test_get_result_per_share_last_three_years(self):
soup = scrap.get_soup_code_from_file("data/bo_bilanz_guv.html")
result_15, result_16, result_17 = parse.get_result_per_share_last_three_years(
soup)
asserted_result_2015 = 3.3
asserted_result_2016 = 5.08
asserted_result_2017 = 6.69
self.assertEqual(asserted_result_2015, result_15)
self.assertEqual(asserted_result_2016, result_16)
self.assertEqual(asserted_result_2017, result_17)
def write_stock_balance_data_to_db():
stock_list = db.get_list(table=cst.TABLE_STOCKS, columns=cst.COLUMN_URI)
file_list = [
cst.PATH_STOCK_BALANCE + stock[:-6] + cst.HTML_EXTENSION
for stock in stock_list
]
for file in file_list:
stock_uri = file.split("/")[-1][:-5] + "-Aktie"
stock_balance_soup = scrap.get_soup_code_from_file(file)
if stock_balance_soup is None:
logger.warning(
"Write Stock Balance Data to DB: Soup is None for %s" %
str(file))
continue
if not parse.is_data_available(stock_balance_soup):
logger.warning(
"Write Stock Balance Data to DB: No Data Available for %s" %
str(file))
continue
try:
earnings_after_tax = parse.get_current_value_of_attribute(
stock_balance_soup, cst.TEXT_RESULT_AFTER_TAX)
operative_result = parse.get_current_value_of_attribute(
stock_balance_soup, cst.TEXT_OPERATIVE_RESULT)
sales_revenue = parse.get_current_value_of_attribute(
stock_balance_soup, cst.TEXT_SALES_REVENUE)
balance = parse.get_current_value_of_attribute(
stock_balance_soup, cst.TEXT_BALANCE)
equity_capital = parse.get_current_value_of_attribute(
stock_balance_soup, cst.TEXT_EQUITY_CAPITAL)
eps_m3, eps_m2, eps_m1 = parse.get_result_per_share_last_three_years(
stock_balance_soup)
db.upsert_item(
table=cst.TABLE_COMPANY_DATA,
primary_keys=[cst.COLUMN_STOCK_URI, cst.COLUMN_DATE],
current_date=date.get_current_date(),
stock_uri=stock_uri,
equity_capital=equity_capital,
earnings_after_tax=earnings_after_tax,
operative_result=operative_result,
sales_revenue=sales_revenue,
balance=balance,
eps_m3=eps_m3,
eps_m2=eps_m2,
eps_m1=eps_m1)
except:
logger.exception(
"Write Stock Balance Data to DB: Exception for stock: %s" %
stock_uri)
continue
def test_stock_sectors(self):
soup = scrap.get_soup_code_from_file("data/bo_sap-aktie.html")
link_items = parse.get_sectors(soup)
asserted_indizes_values = [
"Informationstechnologie",
"IT-Dienstleister",
"Server-/ Großrechner (Software)",
"Software",
]
self.assertEqual(asserted_indizes_values, link_items)
soup_2 = scrap.get_soup_code_from_file("data/bo_ab_inbev-aktie.html")
link_items_2 = parse.get_sectors(soup_2)
asserted_indizes_values_2 = ["Getränke / Tabak"]
self.assertEqual(asserted_indizes_values_2, link_items_2)
soup3 = scrap.get_soup_code_from_file("data/bo_3i-Aktie_overview.html")
link_items3 = parse.get_sectors(soup3)
asserted_indizes_values3 = ["Finanzdienstleister"]
self.assertEqual(asserted_indizes_values3, link_items3)
def test_get_historic_prices_from_overview(self):
soup = scrap.get_soup_code_from_file("data/bo_3i-Aktie_overview.html")
table_list = parse.get_historic_stock_prices_from_overview(soup)
asserted_list = [
["29.11.2018", "9,87"],
["28.11.2018", "9,69"],
["27.11.2018", "9,75"],
["26.11.2018", "9,62"],
["23.11.2018", "9,44"],
]
self.assertEqual(asserted_list, table_list)
def test_index_stocks_list_extract(self):
soup = scrap.get_soup_code_from_file("data/bo_index_stocks.html")
stocks_table = soup.find_all("div", {"id": "index-list-container"})
table_list = parse.extract_index_stocks_to_list(stocks_table)
asserted_list = [
["adidas", "adidas-Aktie"],
["Allianz", "allianz-Aktie"],
["BASF", "basf-Aktie"],
["Bayer", "bayer-Aktie"],
["Beiersdorf", "beiersdorf-Aktie"],
]
self.assertEqual(table_list[:5], asserted_list)
def test_get_future_dates(self):
soup = scrap.get_soup_code_from_file("data/bo_termine.html")
result = parse.get_future_dates(soup)
asserted_result = [
["adidas AG", "2018-11-07", "Quartalszahlen", "Future"],
["adidas AG", "2019-03-07", "Quartalszahlen", "Future"],
["adidas AG", "2019-05-02", "Quartalszahlen", "Future"],
["adidas AG", "2019-05-09", "Hauptversammlung", "Future"],
["adidas AG", "2019-08-08", "Quartalszahlen", "Future"],
["adidas AG", "2019-11-07", "Quartalszahlen", "Future"],
]
self.assertEqual(asserted_result, result)
def test_get_stock_table_of_index(self):
soup = scrap.get_soup_code_from_file("data/bo_FTSE_100.html")
asserted_stock_list_first = [
["3i", "3i-Aktie"],
["Admiral Group", "admiral_group-Aktie"],
["Anglo American", "anglo_american-Aktie"],
]
asserted_stock_list_last = [
["WPP 2012", "wpp_2012-Aktie"],
["WPP 2012", "wpp_2012-Aktie"],
]
stock_list = parse.get_stock_list_of_single_index(soup)
self.assertEqual(asserted_stock_list_first, stock_list[:3])
self.assertEqual(asserted_stock_list_last, stock_list[-2:])
def write_index_contents_from_html_to_db():
index_list = db.get_list(
table=cst.TABLE_INDIZES,
columns=cst.COLUMN_URI,
condition=[cst.COLUMN_ACTIVE, 1],
)
file_list = [
cst.PATH_INDEX_CONTENT + index + cst.HTML_EXTENSION
for index in index_list
]
for file in file_list:
index_content_soup = scrap.get_soup_code_from_file(file)
stock_list = parse.get_stock_list_of_single_index(index_content_soup)
index_uri = file.split("/")[-1][:-5]
db.write_index_content_list_to_db(stock_list, index_uri)
return True
def write_stock_histories_from_html_to_db():
stock_list = db.get_list(table=cst.TABLE_STOCKS, columns=cst.COLUMN_URI)
file_list = [
cst.PATH_STOCK_HISTORY + stock[:-6] + cst.HTML_EXTENSION
for stock in stock_list
]
for file in file_list:
stock_history_soup = scrap.get_soup_code_from_file(file)
if stock_history_soup is None:
logger.warning(
"Write Stock History Data to DB: Stock History Soup is None for %s"
% str(file))
continue
stock_history_list = parse.get_historic_prices_from_history(
stock_history_soup)
stock_uri = file.split("/")[-1][:-5] + "-Aktie"
db.write_stock_history_to_db(stock_history_list, stock_uri)
def write_stock_overview_data_to_db():
stock_list = db.get_list(table=cst.TABLE_STOCKS, columns=cst.COLUMN_URI)
file_list = [
cst.PATH_STOCK_OVERVIEW + stock + cst.HTML_EXTENSION
for stock in stock_list
]
for file in file_list:
stock_uri = file.split("/")[-1][:-5]
stock_overview_soup = scrap.get_soup_code_from_file(file)
if stock_overview_soup is None:
logger.warning(
"Write Stock Overview Data to DB: Stock Overview Soup is None for %s"
% str(file))
continue
market_cap = parse.get_market_cap(stock_overview_soup)
if market_cap is None:
logger.warning(
"Write Stock Overview Data to DB: Market Cap is None for %s" %
str(file))
market_cap = -1.00
stock_indices = str(parse.get_listed_indizes(stock_overview_soup))
stock_sectors = str(parse.get_sectors(stock_overview_soup))
market_place = parse.get_market_place(stock_overview_soup)
db.upsert_item(table=cst.TABLE_COMPANY_DATA,
primary_keys=[cst.COLUMN_STOCK_URI, cst.COLUMN_DATE],
stock_uri=stock_uri,
market_cap=market_cap,
current_date=date.get_current_date(),
stock_indices=stock_indices,
stock_sectors=stock_sectors)
db.update_item(table=cst.TABLE_STOCKS,
primary_keys=cst.COLUMN_URI,
uri=stock_uri,
market_place=market_place)
stock_history_list = parse.get_historic_stock_prices_from_overview(
stock_overview_soup)
if stock_history_list is not None:
db.write_stock_overview_history_to_db(stock_history_list,
stock_uri)
def write_index_histories_from_html_to_db():
index_list = db.get_list(
table=cst.TABLE_INDIZES,
columns=cst.COLUMN_URI,
condition=[cst.COLUMN_ACTIVE, 1],
)
file_list = [
cst.PATH_INDEX_HISTORY + index + cst.HTML_EXTENSION
for index in index_list
]
for file in file_list:
index_history_soup = scrap.get_soup_code_from_file(file)
index_history_list = parse.get_historic_prices_from_history(
index_history_soup)
index_uri = file.split("/")[-1][:-5]
try:
db.write_index_history_to_db(index_history_list, index_uri)
except:
logger.exception(
"Unhandled Exceptions at parse.write_index_histories_from_html_to_db"
)
def test_get_stock_indices(self):
soup = scrap.get_soup_code_from_file("data/bo_bechtle-Aktie.html")
indices_list = parse.get_listed_indizes(soup)
asserted_indices = [
"TecDAX",
"MDAX",
"LMDAX",
"LTecDAX",
"HDAX",
"Technology All Share",
"Prime All Share",
"CDAX",
"TecDAX Kursindex",
"MDAX Kursindex",
"BX Swiss - EMEA",
"DAXglobal Sarasin Sustainability Germany Index EUR",
"QIX Deutschland",
"DAXglobal Sarasin Sustainability Germany",
"Schatten-Index-TecDAX",
"Schatten-Index-SDAX",
]
self.assertEqual(asserted_indices, indices_list)
def test_get_data_available_info(self):
soup = scrap.get_soup_code_from_file("data/no_history_data.html")
self.assertEqual(False, scrap.is_data_available(soup))
def test_get_market_place(self):
soup3 = scrap.get_soup_code_from_file(
"data/bo_aes-Aktie_overview.html")
market_place3 = parse.get_market_place(soup3)
asserted_market_place3 = "FSE"
self.assertEqual(asserted_market_place3, market_place3)
def test_get_historic_prices_from_overview_without_content(self):
soup = scrap.get_soup_code_from_file(
"data/bo_overview_abertis-Aktie.html")
table_list = parse.get_historic_stock_prices_from_overview(soup)
self.assertIsNone(table_list)
def test_get_closing_price_from_date_before(self):
soup = scrap.get_soup_code_from_file("data/bo_kurse.html")
closing_price = parse.get_closing_price_from_date_before(
soup, "09.08.2018")
asserted_price = 190.55
self.assertEqual(asserted_price, closing_price)
def test_get_sales_revenue(self):
soup = scrap.get_soup_code_from_file("data/bo_bilanz_guv.html")
result = parse.get_current_value_of_attribute(soup, "Umsatzerlöse")
asserted_result = 21218
self.assertEqual(asserted_result, result)
def test_pagination_getting(self):
soup = scrap.get_soup_code_from_file(
"data/bo_index_stocks_pagination.html")
highest_pagination = scrap.get_max_page(soup)
self.assertEqual(3, highest_pagination)
def test_get_total_assets(self):
soup = scrap.get_soup_code_from_file("data/bo_bilanz_guv.html")
result = parse.get_current_value_of_attribute(soup, "Bilanzsumme")
asserted_result = 14522
self.assertEqual(asserted_result, result)
def test_get_equity_capital_new(self):
soup = scrap.get_soup_code_from_file("data/bo_bilanz_guv.html")
result = parse.get_current_value_of_attribute(soup, "Eigenkapital")
asserted_result = 6435
self.assertEqual(asserted_result, result)
def test_get_last_quarterly_figures_date(self):
soup = scrap.get_soup_code_from_file("data/bo_termine.html")
result = parse.get_last_quarterly_figures_date(
soup=soup, current_date=date.string_to_date("10.10.2018"))
asserted_result = date.string_to_date("09.08.18")
self.assertEqual(asserted_result, result)
