def date_finder(text):
date =""
date_pattern = '%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}'
matches = list(datefinder.find_dates(s))
match_date = re.search('\d{4}-\d{2}-\d{2}', s)
try:
print "====using dateutil"
for i in s.splitlines():
d = parser.parse(i)
print(d.strftime("%Y-%m-%d"))
except Exception as e:
print e
try:
print "====pygrok==="
grok = Grok(date_pattern)
print(grok.match(s))
except Exception as e:
print e
try:
print "====using date==="
if len(matches) > 0:
date = matches[0]
print date
else:
print 'No dates found'
except Exception as e:
print e
try:
print "====using date==="
date = datetime.datetime.strptime(match_date.group(), '%Y-%m-%d').date()
print date
except Exception as e:
print e
try:
print "====using Chunkgrams==="
chunkGram = r"""NE:{<NNP>+<CD>}"""
chunkParser = nltk.RegexpParser(chunkGram)
sentences = nltk.sent_tokenize(text)
tokenized_sentences = [nltk.word_tokenize(sentence.strip()) for sentence in sentences]
tagged_sentences = [nltk.pos_tag(i) for i in tokenized_sentences]
chunked_sentences = [chunkParser.parse(i) for i in tagged_sentences]
entity_names = []
for tree in chunked_sentences:
entity_names.extend(extract_entity_names(tree))
print entity_names
except Exception as e:
print e
try:
print "===using pydatum=="
datum = Datum()
print (datum.from_iso_date_string(text))
except Exception as e:
print e
try:
print "===using dateparser=="
date = search_dates(text.decode('ascii','ignore'))
print date
except Exception as e:
print e
def get_todays_file_path(self):
""" path to today's cached file """
datum = Datum()
datum.today()
today = datum.get_iso_date_string()
return self.__get_rate_file_path(today)
def map_entity(self, entity: dal.Price) -> PriceModel:
""" Map the price entity """
if not entity:
return None
result = PriceModel()
result.currency = entity.currency
# date/time
dt_string = entity.date
format_string = "%Y-%m-%d"
if entity.time:
dt_string += f"T{entity.time}"
format_string += "T%H:%M:%S"
price_datetime = datetime.strptime(dt_string, format_string)
result.datum = Datum()
result.datum.from_datetime(price_datetime)
assert isinstance(result.datum, Datum)
#result.namespace = entity.namespace
#result.symbol = entity.symbol
result.symbol = SecuritySymbol(entity.namespace, entity.symbol)
# Value
value = Decimal(entity.value) / Decimal(entity.denom)
result.value = Decimal(value)
return result
def __get_model_for_portfolio_value(input_model: PortfolioValueInputModel
) -> PortfolioValueViewModel:
""" loads the data for portfolio value """
result = PortfolioValueViewModel()
result.filter = input_model
ref_datum = Datum()
ref_datum.from_datetime(input_model.as_of_date)
ref_date = ref_datum.end_of_day()
result.stock_rows = []
with BookAggregate() as svc:
book = svc.book
stocks_svc = svc.securities
if input_model.stock:
symbols = input_model.stock.split(",")
stocks = stocks_svc.get_stocks(symbols)
else:
stocks = stocks_svc.get_all()
for stock in stocks:
row: StockViewModel = portfoliovalue.get_stock_model_from(
book, stock, as_of_date=ref_date)
if row and row.balance > 0:
result.stock_rows.append(row)
return result
def get_yesterdays_file_path(self):
""" Full path to the today's rates file. """
datum = Datum()
datum.yesterday()
yesterday = datum.get_iso_date_string()
return self.__get_rate_file_path(yesterday)
def add_price_for_yesterday(session):
""" Create a price entry for test(s) """
value = Decimal("1.0548")
app = PriceDbApplication(session=session)
datum = Datum()
datum.yesterday()
symbol = SecuritySymbol("VANGUARD", "BOND")
model = PriceModel()
model.currency = "AUD"
model.datum = datum
model.symbol = symbol
model.value = value
app.add_price(model)
# make sure that the price is there
first = app.price_repo.query.first()
assert first
yesterday_str = datum.to_iso_date_string()
assert first.date == yesterday_str
assert first.currency == "AUD"
assert first.value == 10548
assert first.denom == 10000
def download(self, symbol: SecuritySymbol, currency: str):
""" Download price.
The currency argument is ignored and is always AUD.
"""
if symbol.namespace != "Vanguard".upper():
raise ValueError(
f"Only Vanguard namespace is handled by this agent. Requested {symbol.namespace}:{symbol.mnemonic}!"
)
fund_data = self.__load_fund_data()
fund_id = self.fund_map[str(symbol)]
fund_info = self.__get_fund_price(fund_data, fund_id)
# self.logger.debug(f"{price}")
result = PriceModel()
date_format = "%d %b %Y"
price_datetime = datetime.strptime(fund_info.date, date_format)
result.datum = Datum()
result.datum.from_datetime(price_datetime)
result.symbol = SecuritySymbol("VANGUARD", symbol.mnemonic)
value = fund_info.value.strip("$")
result.value = Decimal(value)
result.currency = "AUD"
return result
def parse_price(self, html: str) -> PriceModel:
""" parse html to get the price """
from bs4 import BeautifulSoup
from pydatum import Datum
result = PriceModel()
soup = BeautifulSoup(html, 'html.parser')
# Price value
price_el = soup.find(id='last-price-value')
if not price_el:
logging.debug(f"Received from mstar: {html}")
raise ValueError("No price info found in returned HTML.")
value = price_el.get_text().strip()
try:
result.value = Decimal(value)
except InvalidOperation:
message = f"Could not parse price value {value}"
print(message)
self.logger.error(message)
return None
# The rest
date_str = soup.find(id="asOfDate").get_text().strip()
date_val = datetime.strptime(date_str, "%m/%d/%Y %H:%M:%S")
result.datum = Datum()
result.datum.from_datetime(date_val)
# tz_str = soup.find(id="timezone").get_text().strip()
currency = soup.find(id="curency").get_text().strip()
result.currency = currency
return result
def main(args):
""" Main entry point. Useful for calling from cli script or unit tests """
from pydatum import Datum
from gnucash_portfolio.reports import portfolio_value
from gnucash_portfolio.reports.portfolio_models import PortfolioValueInputModel
parameters = PortfolioValueInputModel()
today = Datum()
today.today()
parameters.as_of_date = today.value
model = portfolio_value.run(parameters)
# Display
print_row("security", "cost", "balance", "gain/loss", "%", "income", "inc.12m", "%")
print("-" * 120)
rows = sorted(model.stock_rows, key=lambda row: f"{row.exchange}:{row.symbol}")
for row in rows:
col1 = f"{row.exchange}:{row.symbol}"
col2 = f"{row.shares_num:,} @ {row.avg_price:,.2f} {row.currency} = {row.cost:>7,.2f}"
col3 = f"@ {row.price:,.2f} = {row.balance:>9,.2f}"
col4 = f"{row.gain_loss:,.2f}"
col5 = f"{row.gain_loss_perc:,.2f}%"
col6 = f"{row.income:,.2f}"
col7 = f"{row.income_last_12m:,.2f}"
col8 = f"{row.income_last_12m_perc:,.2f}"
print_row(col1, col2, col3, col4, col5, col6, col7, col8)
def test_from_iso_date_string():
str_value = "2017-08-23"
datum = Datum()
datum.from_iso_date_string(str_value)
assert datum.value.day == 23
assert datum.value.month == 8
assert datum.value.year == 2017
def test_start_of_day():
datum = Datum()
datum.start_of_day()
actual = datum.value
assert actual.hour == 0
assert actual.minute == 0
assert actual.second == 0
def get_end_balance(self, after: date) -> Decimal:
""" Calculates account balance """
# create a new date without hours
#date_corrected = datetimeutils.end_of_day(after)
datum = Datum()
datum.from_date(after)
datum.end_of_day()
#log(DEBUG, "getting balance on %s", date_corrected)
return self.get_balance_on(datum.value)
def __init__(self):
from pydatum import Datum
today = Datum()
today.today()
self.as_of_date: datetime = today.value
# Only the stock symbols, without exchange, comma-separated.
self.stock: str = ""
def __init__(self):
from decimal import Decimal
from pydatum import Datum
# self.datetime: datetime = None
self.datum: Datum = Datum()
self.symbol: SecuritySymbol = SecuritySymbol("", "")
self.value: Decimal = Decimal(0)
self.currency: str = None
def get_quantity(self) -> Decimal:
"""
Returns the number of shares for the given security.
It gets the number from all the accounts in the book.
"""
from pydatum import Datum
# Use today's date but reset hour and lower.
today = Datum()
today.today()
today.end_of_day()
return self.get_num_shares_on(today.value)
def get_transactions(self, date_from: datetime, date_to: datetime) -> List[Transaction]:
""" Returns account transactions """
assert isinstance(date_from, datetime)
assert isinstance(date_to, datetime)
# fix up the parameters as we need datetime
dt_from = Datum()
dt_from.from_datetime(date_from)
dt_from.start_of_day()
dt_to = Datum()
dt_to.from_datetime(date_to)
dt_to.end_of_day()
query = (
self.book.session.query(Transaction)
.join(Split)
.filter(Split.account_guid == self.account.guid)
.filter(Transaction.post_date >= dt_from.date, Transaction.post_date <= dt_to.date)
.order_by(Transaction.post_date)
)
return query.all()
def test_add_days():
datum = Datum()
datum.today()
datum.start_of_day()
today = datetime.today()
#day = today.day + 1
tomorrow = today + relativedelta(days=1)
tomorrow = tomorrow.replace(hour=0, minute=0, second=0, microsecond=0)
actual = datum.add_days(1)
assert actual == tomorrow
def get_start_balance(self, before: date) -> Decimal:
""" Calculates account balance """
assert isinstance(before, datetime)
# create a new date without hours
datum = Datum()
datum.from_date(before)
#date_corrected = datetimeutils.start_of_day(before)
# now subtract 1 second.
#date_corrected -= timedelta(seconds=1)
#log(DEBUG, "getting balance on %s", date_corrected)
datum.yesterday()
datum.end_of_day()
return self.get_balance_on(datum.value)
def main(args=None):
""" Show the list of favorite accounts """
print("Favourite accounts:\n")
book = BookAggregate()
favourites = book.accounts.get_favourite_account_aggregates()
favourites = sorted(favourites,
key=lambda aggregate: aggregate.account.name)
for aggregate in favourites:
#balance = aggregate.get_balance()
today = Datum()
today.today()
balance = aggregate.get_balance_on(today.value)
currency = aggregate.account.commodity.mnemonic
print(f"{aggregate.account.name:<25}{balance:>10,.2f} {currency}")
def add_prices_for_yesterday_and_today(session, symbol: SecuritySymbol):
""" Add prices for the same symbol for yesterday and today """
app = PriceDbApplication(session)
assert isinstance(symbol, SecuritySymbol)
price = PriceModel()
# Create price for today
price.datum = Datum()
price.currency = "EUR"
price.symbol = symbol
price.value = Decimal("150.13")
app.add_price(price)
# Create price for yesterday
price.datum.yesterday()
price.value = Decimal("50.28")
app.add_price(price)
def parse_price(self, html: str) -> PriceModel:
""" parse html to get the price """
from bs4 import BeautifulSoup
result = PriceModel()
soup = BeautifulSoup(html, 'html.parser')
overview = soup.find(id='overviewQuickstatsDiv')
# Price
price_el = overview.find('table').find_all('tr')[1].find_all('td')[2]
if not price_el:
logging.debug(f"Received from mstar: {html}")
raise ValueError("No price info found in returned HTML.")
value = price_el.get_text().strip()
# Currency
currency = value[:3]
result.currency = currency
value = value[3:].strip()
# decimal separator
value = value.replace(',', '.')
# parse
try:
result.value = Decimal(value)
except InvalidOperation:
message = f"Could not parse price value {value}"
print(message)
self.logger.error(message)
return None
# Date
date_el = overview.find('table').find_all('tr')[1].find_all(
'td')[0].find('span')
#date_str = soup.find(id="asOfDate").get_text().strip()
date_str = date_el.get_text().strip()
date_val = datetime.strptime(date_str, "%d.%m.%Y")
result.datum = Datum()
result.datum.from_datetime(date_val)
return result
def parse_price(self, json_str: str) -> PriceModel:
''' Get the price from HTML '''
import json
from decimal import Decimal
from datetime import datetime
from pydatum import Datum
content = json.loads(json_str)
result = PriceModel()
price_str = str(content['lastPrice'])
result.value = Decimal(price_str)
result.currency = self.currency
date_str = content['timestampLastPrice']
date_val = datetime.fromisoformat(date_str)
result.datum = Datum()
result.datum.from_datetime(date_val)
return result
def test_latest_date(session):
"""
Test fetching the latest price.
The date is always today, even if the latest price is not from today!
"""
# Preparation
add_price_for_yesterday(session)
# Fetch the latest price for xy
app = PriceDbApplication(session=session)
symbol = SecuritySymbol("VANGUARD", "BOND")
latest_price = app.get_latest_price(symbol)
assert latest_price
yesterday = Datum()
yesterday.yesterday()
yesterday.start_of_day()
yesterday_str = yesterday.to_iso_date_string()
assert latest_price.datum.to_iso_date_string() == yesterday_str
def parse_date(self, date_str: str):
''' parse date/time '''
from dateutil.parser import parse
from dateutil import tz
from pydatum import Datum
# Can be "19 Jul 2019" or "6:06am EDT".
if "EDT" in date_str:
# the format is "6:06am EDT"
from_zone = tz.gettz('US/Eastern')
#est = tz.gettz('EST')
to_zone = tz.tzlocal()
parsed_date = parse(date_str)
date_val = parsed_date.replace(tzinfo=from_zone).astimezone(
tz=to_zone)
else:
# the date format is "19 Jul 2019"
#date_val = datetime.strptime(date_str, "%d %b %Y")
date_val = parse(date_str)
datum = Datum()
result = datum.from_datetime(date_val)
return result
def get_balance(self):
""" Current account balance """
on_date = Datum()
on_date.today()
return self.get_balance_on(on_date.value)
def get_next_occurrence(tx: ScheduledTransaction) -> date:
""" Calculates the next occurrence date for scheduled transaction.
Mimics the recurrenceNextInstance() function from GnuCash.
Still not fully complete but handles the main cases I use. """
# Reference documentation:
# https://github.com/MisterY/gnucash-portfolio/issues/3
# Preparing ref day is an important part before the calculation.
# It should be:
# a) the last occurrence date + 1, or
# b) the recurrence start date - 1.
# because the refDate is the date from which the due dates are being calculated. To include
# the ones starting today, we need to calculate from the day before.
ref_datum: Datum = Datum()
if tx.last_occur:
#ref_datum.set_value(tx.last_occur)
ref_datum.from_date(tx.last_occur)
ref_datum.add_days(1)
else:
ref_datum.from_date(tx.recurrence.recurrence_period_start)
ref_datum.subtract_days(1)
ref_date: datetime = ref_datum.value
# today = datetimeutils.today_date()
#today = Datum().today()
# skipped schedule
#if ref_date > today:
# ref_date = today
###########################################################
# The code below mimics the function
# recurrenceNextInstance(const Recurrence *r, const GDate *refDate, GDate *nextDate)
# https://github.com/Gnucash/gnucash/blob/115c0bf4a4afcae4269fe4b9d1e4a73ec7762ec6/libgnucash/engine/Recurrence.c#L172
start_date: Datum = Datum()
#start_date: datetime = tx.recurrence.recurrence_period_start
start_date.from_date(tx.recurrence.recurrence_period_start)
if ref_date < start_date.value:
# If the occurrence hasn't even started, the next date is the start date.
# this should also handle the "once" type in most cases.
return start_date.value.date()
# start at refDate.
next_date: Datum = Datum()
# next_date: datetime = ref_date
next_date.from_datetime(ref_date)
# last_date: datetime = tx.last_occur
# print(tx.name, base_date, tx.recurrence.recurrence_period_start,
# tx.recurrence.recurrence_mult, tx.recurrence.recurrence_period_type)
# /* Step 1: move FORWARD one period, passing exactly one occurrence. */
mult: int = tx.recurrence.recurrence_mult
period: str = tx.recurrence.recurrence_period_type
wadj = tx.recurrence.recurrence_weekend_adjust
# Not all periods from the original file are included at the moment.
if period in ([
RecurrencePeriod.YEAR.value, RecurrencePeriod.MONTH.value,
RecurrencePeriod.END_OF_MONTH.value
]):
if period == RecurrencePeriod.YEAR.value:
mult *= 12
# handle weekend adjustment here.
## Takes care of short months.
# next_weekday = datetimeutils.get_day_name(next_date)
next_weekday = next_date.get_day_name()
if wadj == WeekendAdjustment.BACK.value and (period in ([
RecurrencePeriod.YEAR.value, RecurrencePeriod.MONTH.value,
RecurrencePeriod.END_OF_MONTH.value
]) and (next_weekday == "Saturday" or next_weekday == "Sunday")):
# "Allows the following Friday-based calculations to proceed if 'next'
# is between Friday and the target day."
days_to_subtract = 1 if next_weekday == "Saturday" else 2
# next_date = datetimeutils.subtract_days(next_date, days_to_subtract)
next_date.subtract_days(days_to_subtract)
if wadj == WeekendAdjustment.BACK.value and (period in ([
RecurrencePeriod.YEAR.value, RecurrencePeriod.MONTH.value,
RecurrencePeriod.END_OF_MONTH.value
]) and next_weekday == "Friday"):
next_date = handle_friday(next_date, period, mult, start_date)
# Line 274.
temp_date = next_date.clone()
if (temp_date.is_end_of_month() or
(period
in [RecurrencePeriod.MONTH.value, RecurrencePeriod.YEAR.value] and
(next_date.get_day() >= start_date.get_day()))):
# next_date = datetimeutils.add_months(next_date, mult)
next_date.add_months(mult)
# Set at end of month again (?!)
#next_date = datetimeutils.get_end_of_month(next_date)
else:
# one fewer month fwd because of the occurrence in this month.
next_date.add_months(mult - 1)
# elif period == "once":
# next_date = tx.recurrence.recurrence_period_start
elif period == RecurrencePeriod.DAY.value:
logging.warning("daily not handled")
else:
logging.info(f"recurrence not handled: {period}")
#######################
# Step 2
# "Back up to align to the base phase. To ensure forward
# progress, we never subtract as much as we added (x % mult < mult)"
if period in ([
RecurrencePeriod.YEAR.value, RecurrencePeriod.MONTH.value,
RecurrencePeriod.END_OF_MONTH.value
]):
n_months = (12 * (next_date.get_year() - start_date.get_year()) +
(next_date.get_month() - start_date.get_month()))
next_date.subtract_months(n_months % mult)
# dim
days_in_month = datetimeutils.get_days_in_month(
next_date.get_year(), next_date.get_month())
# Handle adjustment for 3 ways.
if (period == RecurrencePeriod.END_OF_MONTH.value
or next_date.get_day() >= days_in_month):
# Set to last day of the month.
next_date.set_day(days_in_month)
else:
# Same day as the start.
next_date.set_day(start_date.get_day())
# Adjust for dates on the weekend.
if (period == RecurrencePeriod.YEAR.value
or period == RecurrencePeriod.MONTH.value
or period == RecurrencePeriod.END_OF_MONTH.value):
weekday = next_date.get_day_name()
if weekday == "Saturday" or weekday == "Sunday":
if wadj == WeekendAdjustment.BACK.value:
next_date.subtract_days(1 if weekday == "Saturday" else 2)
elif wadj == WeekendAdjustment.FORWARD.value:
next_date.add_days(2 if weekday == "Saturday" else 1)
return next_date.value.date()
def test_instantiation():
""" Test creation of the object """
actual = Datum()
assert actual is not None
def get_stock_model_from(book: Book, commodity: Commodity,
as_of_date: date) -> StockViewModel:
""" Parses stock/commodity and returns the model for display """
from decimal import Decimal
from pydatum import Datum
svc = SecurityAggregate(book, commodity)
model = StockViewModel()
model.exchange = commodity.namespace
model.symbol = commodity.mnemonic
model.shares_num = svc.get_num_shares_on(as_of_date)
# Ignore 0-balance
if model.shares_num == 0:
return None
model.avg_price = svc.get_avg_price()
# Last price
price_svc = PricesAggregate(book)
# last_price: Price = price_svc.get_price_as_of(commodity, as_of_date)
last_price: PriceModel = price_svc.get_latest_price(commodity)
if last_price is not None:
model.price = last_price.value
else:
model.price = Decimal(0)
# currency
if model.price:
model.currency = last_price.currency
# Cost
model.cost = model.shares_num * model.avg_price
# Balance (current value)
if model.shares_num > 0 and model.price:
model.balance = model.shares_num * model.price
else:
model.balance = Decimal(0)
# Gain/Loss
model.gain_loss = model.balance - model.cost
# Gain/loss percentage
gain_loss_perc = 0
if model.cost:
gain_loss_perc = abs(model.gain_loss) * 100 / model.cost
if model.gain_loss < 0:
gain_loss_perc *= -1
model.gain_loss_perc = gain_loss_perc
# Income
income = symbol_dividends.get_dividend_sum_for_symbol(book, model.symbol)
model.income = float(income)
#income_12m = symbol_dividends.
start = Datum()
start.subtract_months(12)
end = Datum()
model.income_last_12m = svc.get_income_in_period(start, end)
if model.balance > 0 and model.income_last_12m > 0:
model.income_last_12m_perc = model.income_last_12m * 100 / model.balance
else:
model.income_last_12m_perc = Decimal(0)
return model
def test_clone():
one = Datum()
two = one.clone()
assert one.value == two.value
def test_init_value():
""" The initial value should be now """
datum = Datum()
now = datetime.now()
assert now == datum.value
