def dump_stock(stock: Stock, stockStorage: StockStorage):
main_file = stockStorage.getStoragePath("profil", "html")
storage_repository = stockStorage.storage_repository
dl.download(WEBSITE + "/aktien/" + stock.name, main_file, storage_repository)
dl.download(f"%s/bilanz_guv/%s" % (WEBSITE, stock.name), stockStorage.getStoragePath("bilanz_guv", "html"), storage_repository)
dl.download(f"%s/schaetzungen/%s" % (WEBSITE, stock.name), stockStorage.getStoragePath("schaetzungen", "html"), storage_repository)
dl.download(f"%s/termine/%s" % (WEBSITE, stock.name), stockStorage.getStoragePath("termine", "html"), storage_repository)
dl.download(f"%s/analysen/%s-analysen" % (WEBSITE, stock.name), stockStorage.getStoragePath("analysen", "html"), storage_repository)
download_history(stockStorage)
def get_allianz_stock_storage(get_history=False, date=datetime.now()):
indexGroup = IndexGroup("DE0008469008", "DAX", "DAX", "onvista")
index_storage = IndexStorage("resources", indexGroup,
date=date,
get_history=get_history)
stock = Stock("DE0008404005", "Allianz", indexGroup)
return StockStorage(index_storage, stock)
def test_base_path_of_stock(self):
# given:
index_group = IndexGroup("isin", "index_name", "source_id", "source")
stock = Stock("stock_id", "stock_name", index_group)
date = datetime.strptime("2018-01-01", "%Y-%m-%d")
index_storage = IndexStorage("/tests/dump",
index_group,
date,
get_history=False)
# when:
stock_storage = StockStorage(index_storage, stock)
base_path = stock_storage.getDatedPath()
# then:
self.assertEqual("/tests/dump/index_name/2018-01-01/", base_path)
def get_vw_stock_storage(get_history=False, date=datetime.now()):
indexGroup = IndexGroup("DE0008469008", "DAX", "DAX", "onvista")
index_storage = IndexStorage("resources", indexGroup,
date=date,
get_history=get_history)
stock = Stock("DE0007664039", "Volkswagen-VZ", indexGroup)
return StockStorage(index_storage, stock)
def download_history(stock_name: str, stockStorage: StockStorage):
path = stockStorage.getStoragePath("history", "html")
content = stockStorage.storage_repository.load(path)
if content:
soup = BeautifulSoup(content, 'html.parser')
selectbox = soup.find("div", {"id": "exchangesLayerHs"})
if not selectbox:
return
options = selectbox.findAll("a")
if options is None:
print("unable to find historical data for stock {}".format(
stock_name))
def create_StockExchangeOpt(opt):
volume = opt.find("span").get_text().strip()
volume = volume.replace(" Stk.", "")
volume = "0" if volume is None or volume == "" else volume
volume = volume.replace(".", "")
notation = opt.get('href').split("=")[1]
opt.find("span").decompose()
name = opt.get_text().strip()
return StockExchangeOpt(option=opt,
name=name,
notation=notation,
volume=int(volume))
def is_valid_exchange_option(opt: StockExchangeOpt,
ref_index: str) -> bool:
if ref_index == "TecDAX":
return opt.name != "Swiss Exchange"
return True
ref_index = stockStorage.stock.indexGroup.name
seo = seq(options) \
.map(create_StockExchangeOpt) \
.filter(lambda se: is_valid_exchange_option(se, ref_index)) \
.sorted(lambda se: se.volume, reverse=True) \
.first()
if seo:
print(
"download history for '{}' from '{}' stock exchange (volumne: {})"
.format(stock_name, seo.name, seo.volume))
download_history_by_notation(seo.notation, stockStorage)
else:
print("unable to find notation for stock {}".format(stock_name))
def add_historical_eps(stock: Stock, stock_storage: StockStorage):
if not stock_storage.indexStorage.historicalStorage:
stock.historical_eps_current_year = 0
stock.historical_eps_next_year = 0
return
historical_storage = StockStorage(
stock_storage.indexStorage.historicalStorage, stock)
stock.historical_eps_date = historical_storage.indexStorage.date_str
try:
historical_storage.load()
historical_stock = historical_storage.stock
stock.historical_eps_current_year = historical_stock.eps_current_year
stock.historical_eps_next_year = historical_stock.eps_next_year
except FileNotFoundError:
stock.historical_eps_current_year = 0
stock.historical_eps_next_year = 0
def dump_stock(event: dict, context):
data = decode_pubsub_data(event)
logging.info("data: {}".format(data))
source = data["source"]
index_group = new_index_group(data["index_group"])
stock = new_stock(data["stock"], index_group)
date = date_or_now(data)
logging.info("source: {}".format(source))
logging.info("index_group: {}".format(index_group))
logging.info("stock: {}".format(stock))
logging.info("date: {}".format(date))
index_storage = IndexStorage(DUMP_FOLDER, index_group, date=date, storage_repository=S3Repository(BUCKET_NAME))
stock_storage = StockStorage(index_storage, stock, storage_repository=S3Repository(BUCKET_NAME))
downloader = DownloaderFactory.create(source)
downloader.dump_stock(stock, stock_storage)
scraper = ScraperFactory.create(source)
scraper.scrap(stock, stock_storage)
stock_storage.store()
stock_storage.compress()
def write_appointments(appointments, stock_storage: StockStorage):
if len(appointments) > 0:
path = stock_storage.indexStorage.getAppointmentsPath()
csv_file = path + stock_storage.getFilename("company-and-appointments",
"csv")
content = "Date;Topic;\n"
for key, value in appointments.items():
content += "{};{};\n".format(key, value)
stock_storage.storage_repository.store(csv_file, content)
def dump_stock(stock: Stock, stock_storage: StockStorage):
main_file = stock_storage.getStoragePath("profil", "html")
storage_repository = stock_storage.storage_repository
dl.download(WEBSITE + "/aktien/" + stock.stock_id, main_file,
storage_repository)
links = get_links(main_file, storage_repository)
dl.download(WEBSITE + links["Fundamental"],
stock_storage.getStoragePath("fundamental", "html"),
storage_repository)
dl.download(WEBSITE + links["T&S/Historie"],
stock_storage.getStoragePath("history", "html"),
storage_repository,
retry=True)
dl.download(
WEBSITE + links["Profil/Termine"],
stock_storage.getStoragePath("company-and-appointments", "html"),
storage_repository)
download_history(stock.name, stock_storage)
download_ratings(stock.stock_id, stock_storage)
def write_stock_report(stock: Stock, stock_storage: StockStorage,
rating: Rating):
template = Template(filename="libs/templates/stock-rating.html")
stock_before, stock_after = find_stock_next_to(stock,
stock.indexGroup.stocks)
report = template.render(stock=stock,
rating=rating,
source=stock_storage.indexStorage.source,
report_date=stock_storage.indexStorage.date_str,
stock_before=stock_before,
stock_after=stock_after)
stock_storage.storage_repository.store(
stock_storage.getStoragePath("", "html"), report)
def read_existing_appointments(stock_storage: StockStorage):
appointments = {}
path = stock_storage.indexStorage.getAppointmentsPath()
csv_file = path + stock_storage.getFilename("company-and-appointments",
"csv")
content = stock_storage.storage_repository.load(csv_file)
if content:
rows = csv.DictReader(content.splitlines(), delimiter=';')
for row in rows:
date = row["Date"].strip()
topic = row["Topic"].strip()
appointments[date] = topic
return appointments
def scrap_ratings(stock, stock_storage: StockStorage):
filename = stock_storage.getStoragePath("ratings", "html")
ratings = {"kaufen": 0, "halten": 0, "verkaufen": 0}
content = stock_storage.storage_repository.load(filename)
if content:
soup = BeautifulSoup(content, 'html.parser')
for row in soup.findAll("tr"):
columns = row.findAll("td")
type = columns[0].get_text().strip()
count = columns[1]
count.div.decompose()
ratings[type] = int(count.get_text().strip())
stock.ratings = AnalystRatings(ratings["kaufen"], ratings["halten"],
ratings["verkaufen"])
return stock
def download_ratings(stock_id: str, stockStorage: StockStorage):
url = "https://www.onvista.de/news/boxes/aggregated-analyses" \
"?timespan=-1+month&assetType=Stock&assetId=" + stock_id + "&showAllAnalyzesLink=0"
dl.download(url, stockStorage.getStoragePath("ratings", "html"),
stockStorage.storage_repository)
traceback.print_exc()
queue.task_done()
workers = 10
stock_queue = Queue()
for i in range(workers):
worker = Thread(target=thread_body, args=(stock_queue, ))
worker.setDaemon(True)
worker.start()
for stock in indexGroup.stocks:
stock_storage = StockStorage(index_storage, stock)
stock_queue.put((stock, stock_storage))
print('## Start downloading')
stock_queue.join()
print('## Finished downloading')
print('## Start reporting')
RatingEntity = namedtuple('RatingEntity', 'stock, rating')
rating_entities = []
for stock in indexGroup.stocks:
stock_storage = StockStorage(index_storage, stock)
try:
stock_storage.load()
def scrap(stock: Stock, stock_storage: StockStorage, util: OnVistaDateUtil = OnVistaDateUtil()):
with open(stock_storage.getStoragePath("profil", "html"), mode="r") as f:
soup = BeautifulSoup(f, 'html.parser')
currencies = scrap_currencies(soup)
price = scrap_price(soup)
fundamentals = scrap_fundamentals(soup)
company_details = scrap_company_details(soup)
with open(stock_storage.getStoragePath("bilanz_guv", "html"), mode="r") as f:
soup = BeautifulSoup(f, 'html.parser')
bilanz_guv = scrap_bilanz_guv(soup)
with open(stock_storage.getStoragePath("schaetzungen", "html"), mode="r") as f:
soup = BeautifulSoup(f, 'html.parser')
schaetzung = scrap_schaetzung(soup)
with open(stock_storage.getStoragePath("analysen", "html"), mode="r") as f:
soup = BeautifulSoup(f, 'html.parser')
stock.ratings = scrap_analysen(soup)
last_year = util.get_last_year()
current_year = util.get_current_year(estimated=False)
next_year = util.get_next_year(estimated=False)
stock.price = asFloat(price)
stock.field = company_details["Branchen"]
GuV = findIn(bilanz_guv, "GuV")
gewinn = asFloat(GuV["Ergebnis nach Steuer"][last_year])
ebit = asFloat(GuV["Ergebnis vor Steuern"][last_year])
erloes = asFloat(GuV["Umsatzerlöse"][last_year])
bilanz = findIn(bilanz_guv, "Bilanz")
eigenkapital = asFloat(bilanz["Eigenkapital"][last_year])
stock.roi = gewinn / eigenkapital * 100
stock.ebit_margin = ebit / erloes * 100
unternehmenskennzahlen = findIn(bilanz_guv, "Unternehmenskennzahlen")
stock.equity_ratio = asFloat(unternehmenskennzahlen["Eigenkapitalquote in %"][last_year])
stock.per = asFloat(schaetzung["KGV"][current_year])
hist_pers = unternehmenskennzahlen["KGV (Jahresendkurs)"]
per_5_years = stock.per
number_of_year = 1
for year in list(hist_pers.keys())[-4:]:
if hist_pers[year] != "-":
per_5_years += asFloat(hist_pers[year])
number_of_year += 1
stock.per_5_years = (per_5_years / number_of_year)
eps_row_name = "Ergebnis/Aktie (reported)" if ("Ergebnis/Aktie (reported)" in schaetzung) else "Ergebnis/Aktie"
stock.eps_current_year = asFloat(schaetzung[eps_row_name][current_year])
stock.eps_next_year = asFloat(schaetzung[eps_row_name][next_year])
stock.per_fallback = stock.price / stock.eps_current_year if stock.eps_current_year != 0 else 0
stock.market_capitalization = asFloat(fundamentals["Marktkapitalisierung in Mrd. EUR"]) * 1000000000
stock_price_today = 0
stock_price_6month = 0
stock_price_1year = 0
stock.history = History(stock_price_today, stock_price_6month, stock_price_1year)
stock.monthClosings = MonthClosings()
stock.historical_eps_current_year = 0
stock.historical_eps_date = 0
stock.historical_eps_next_year = 0
stock.reaction_to_quarterly_numbers = ReactionToQuarterlyNumbers(0, 0, 0, 0, "")
return stock
def scrap(stock: Stock,
stock_storage: StockStorage,
util: OnVistaDateUtil = OnVistaDateUtil()):
path = stock_storage.getStoragePath("fundamental", "html")
content = stock_storage.storage_repository.load(path)
if content:
soup = BeautifulSoup(content, 'html.parser')
stock.symbol = scrap_symbol(soup)
fundamentals = scrap_fundamentals(soup)
last_year_est = util.get_last_year(estimated=True)
last_cross_year_est = util.get_last_cross_year(estimated=True)
fallback_to_last_year_values = last_year_est in fundamentals[
"Rentabilität"][
"Eigenkapitalrendite"] or last_cross_year_est in fundamentals[
"Rentabilität"]["Eigenkapitalrendite"]
if fallback_to_last_year_values:
last_year = util.get_last_year(min_years=2)
last_cross_year = util.get_last_cross_year(min_years=2)
current_year = util.get_last_year(estimated=True)
current_cross_year = util.get_last_cross_year()
current_cross_year_est = util.get_last_cross_year(estimated=True)
next_year = util.get_current_year()
next_cross_year = util.get_current_cross_year()
else:
last_year = util.get_last_year()
last_cross_year = util.get_last_cross_year()
current_year = util.get_current_year()
current_cross_year = util.get_current_cross_year(estimated=False)
current_cross_year_est = util.get_current_cross_year()
next_year = util.get_next_year()
next_cross_year = util.get_next_cross_year()
stock.price = asFloat(
soup.find("ul", {
"class": "KURSDATEN"
}).find("li").find("span").get_text().strip())
stock.roi = asFloat(
get_for_year(fundamentals["Rentabilität"]["Eigenkapitalrendite"],
[last_year, last_cross_year]))
stock.ebit_margin = asFloat(
get_for_year(fundamentals["Rentabilität"]["EBIT-Marge"],
[last_year, last_cross_year]))
stock.equity_ratio = asFloat(
get_for_year(fundamentals["Bilanz"]["Eigenkapitalquote"],
[last_year, last_cross_year]))
stock.per_5_years = calc_per_5_years(
fundamentals,
[current_year, current_cross_year_est, current_cross_year])
stock.per = asFloat(
get_for_year(
fundamentals["Gewinn"]["KGV"],
[current_year, current_cross_year_est, current_cross_year]))
date = stock_storage.indexStorage.date
if sameDay(date, datetime.now()):
date = date - relativedelta(days=1)
stock_price_today = get_latest_price(stock_storage, date)
stock_price_6month = get_historical_price(
stock_storage, (date - relativedelta(months=6)))
stock_price_1year = get_historical_price(
stock_storage, (date - relativedelta(months=12)))
stock.history = History(stock_price_today, stock_price_6month,
stock_price_1year)
stock.monthClosings = get_month_closings(stock_storage)
stock.eps_current_year = asFloat(
get_for_year(
fundamentals["Gewinn"]["Gewinn pro Aktie in EUR"],
[current_year, current_cross_year_est, current_cross_year]))
stock.per_fallback = stock.price / stock.eps_current_year if stock.eps_current_year != 0 else 0
stock.eps_next_year = asFloat(
get_for_year(fundamentals["Gewinn"]["Gewinn pro Aktie in EUR"],
[next_year, next_cross_year]))
stock.market_capitalization = get_market_capitalization(
fundamentals, last_year, last_cross_year)
stock = scrap_ratings(stock, stock_storage)
add_historical_eps(stock, stock_storage)
add_reaction_to_quarterly_numbers(stock, stock_storage)
return stock