def test_index_group_to_json(self):
# given:
index_group = IndexGroup("isin", "index_name", "source_id", "source")
index_group.stocks = [Stock("stock_id", "stock_name", index_group)]
index_group.history = History(1, 2, 3)
index_group.monthClosings = MonthClosings()
index_group.monthClosings.closings = [0, 0, 0, 0]
def get_allianz_stock_storage(get_history=False, date=datetime.now()):
indexGroup = IndexGroup("DE0008469008", "DAX", "DAX", "onvista")
index_storage = IndexStorage("resources", indexGroup,
date=date,
get_history=get_history)
stock = Stock("DE0008404005", "Allianz", indexGroup)
return StockStorage(index_storage, stock)
def get_vw_stock_storage(get_history=False, date=datetime.now()):
indexGroup = IndexGroup("DE0008469008", "DAX", "DAX", "onvista")
index_storage = IndexStorage("resources", indexGroup,
date=date,
get_history=get_history)
stock = Stock("DE0007664039", "Volkswagen-VZ", indexGroup)
return StockStorage(index_storage, stock)
def scrap_index(indexGroup: IndexGroup, index_storage: IndexStorage):
date = index_storage.date
if sameDay(date, datetime.now()):
date = date - relativedelta(days=1)
index_price_today = get_latest_price(index_storage, date)
index_price_6month = get_historical_price(index_storage,
(date - relativedelta(months=6)))
index_price_1year = get_historical_price(index_storage,
(date - relativedelta(months=12)))
indexGroup.history = History(index_price_today, index_price_6month,
index_price_1year)
indexGroup.monthClosings = get_month_closings(index_storage)
def fromJson(self, json_str: str) -> IndexGroup:
index_json = json.loads(json_str)
# backward compatibilities
isin = index_json["isin"] if "isin" in index_json else index_json["index"]
name = index_json["name"]
sourceID = index_json["sourceId"] if "sourceId" in index_json else name
source = index_json["source"] if "source" in index_json else "onvista"
indexGroup = IndexGroup(isin, name, sourceID, source)
history = index_json["history"]
indexGroup.history = History(history["today"], history["half_a_year"], history["one_year"])
indexGroup.monthClosings = MonthClosings()
indexGroup.monthClosings.closings = index_json["monthClosings"].get("closings")
indexGroup.stocks = list(map(lambda s: Stock(s.id, s.name, indexGroup)), index_json["stocks"])
return indexGroup
def test_base_path_of_index(self):
# given:
index_group = IndexGroup("isin", "index_name", "source_id", "source")
date = datetime.strptime("2018-01-01", "%Y-%m-%d")
# when:
index_storage = IndexStorage("/tests/dump",
index_group,
date,
get_history=False)
base_path = index_storage.getDatedPath()
# then:
self.assertEqual("/tests/dump/index_name/2018-01-01/", base_path)
def test_storage_path_of_index(self):
# given:
index_group = IndexGroup("isin", "index_name", "source_id", "source")
date = datetime.strptime("2018-01-01", "%Y-%m-%d")
# when:
index_storage = IndexStorage("/tests/dump",
index_group,
date,
get_history=False)
storage_path = index_storage.getStoragePath("profile", "html")
# then:
self.assertEqual(
"/tests/dump/index_name/2018-01-01/index_name.source.profile.html",
storage_path)
def createFor(source: str,
indexName: str,
confFile: str = "indexGroup-by-provider.csv") -> IndexGroup:
if not os.path.isfile(confFile):
raise ValueError(f"Config file %s not found" % confFile)
with open(confFile, mode="r", encoding="utf-8") as f:
indexConfig = csv.DictReader(f)
for index in indexConfig:
if index["name"] == indexName and source in index and index[
source] is not None and index[source] != "":
isin = index["isin"]
sourceId = index[source]
return IndexGroup(isin, indexName, sourceId, source)
raise ValueError(
f"Unable to find configuration for index %s from source %s" %
(indexName, source))
def send_scrap_messages(index_group: IndexGroup, date: datetime):
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(PROJECT_ID, STOCK_DUMP_TOPIC_ID)
for stock in index_group.stocks:
data = json.dumps({
"source": index_group.source,
"index_group": index_group.as_dict(),
"stock": stock.as_dict(),
"date": date.isoformat()
}).encode("utf-8")
logging.info("publish data: {}".format(data))
future = publisher.publish(topic_path, data)
# noinspection PyBroadException
try:
future.result()
logging.info("publish {} to pub/sup: {}".format(STOCK_DUMP_TOPIC_ID, stock.name))
except Exception:
logging.exception("unable to publish {} to pub/sup: {}".format(STOCK_DUMP_TOPIC_ID, stock.name))
def new_index_group(data):
return IndexGroup(data["isin"], data["name"], data["sourceId"], data["source"])