def diff_with_backup(predicates, timestamp=afy.settings.start_time, order='first'):
all_backup_transactions = afy.backup.local.load_before(ynab_api.TransactionDetail, timestamp)
matching = utils.multi_filter(predicates, all_backup_transactions)
def key(t):
return t.id
# Get the first copy of the transaction saved
# So that eg default arguments restore state at start of prior run
unique = utils.by(get_unique(matching, key, order), key)
unique_keys = set(unique.keys())
afy.Assistant.download_ynab(transactions=True) # TODO: can I get away with this?
current_transactions = utils.by(
utils.multi_filter(
predicates,
afy.Assistant.transactions.values()),
key)
current_keys = set(current_transactions.keys())
modified = unique_keys.intersection(current_keys)
deleted = unique_keys - current_keys
added = current_keys - unique_keys
utils.log_info(
'Found %s modified, %s deleted, %s added transactions' %
(len(modified), len(deleted), len(added)))
return [list(filter(lambda t: t.id in x, matching)) for x in (modified, deleted, added)]
def driver():
utils.log_debug('driver')
global _driver
install.setup_chromedriver()
try:
if is_alive(_driver):
return _driver
close_orphan_drivers()
options = Options()
assert os.path.exists(settings.chrome_data_dir)
options.add_argument('user-data-dir={}'.format(
settings.chrome_data_dir))
options.add_argument('--disable-extensions')
assert os.path.exists(settings.chromedriver_path)
_driver = webdriver.Chrome(options=options,
executable_path=settings.chromedriver_path)
except BaseException:
quit()
if 'data directory is already in use' in traceback.format_exc():
utils.log_exception_debug()
utils.log_error('Must close Selenium-controlled Chrome.')
if input('Try again? [Y/n]\n').lower() != 'n':
utils.log_info('Trying again')
return driver()
sys.exit()
else:
utils.log_exception()
assert _driver
return _driver
def load(data_type):
utils.log_debug('load', data_type)
assert data_type in data_parsers
target_path = csv_paths[data_type]
try:
if not os.path.exists(target_path) or stale(target_path):
d = gui.driver()
url = 'https://smile.amazon.com/gp/b2b/reports'
if url not in d.current_url:
d.get(url)
d.find_element_by_id('report-use-today').click()
enter_start_date()
d.find_element_by_id('report-type').click()
d.find_element_by_id('report-type').send_keys(data_type)
d.find_element_by_id('report-confirm').click()
path = wait_for_download()
utils.log_debug(path, target_path)
os.rename(path, target_path)
except BaseException:
utils.log_exception_debug()
if 'AttributeError' in traceback.format_exc(): # TODO: only retry on missing csv
utils.log_info('real error', traceback.format_exc())
sys.exit()
if input('One more try? [Y/n]').lower() != 'n':
load(data_type)
list_of_dicts = read(target_path)
utils.log_info('Found %s %s' % (len(list_of_dicts), data_type))
return data_parsers[data_type](list_of_dicts)
def get_order(t, orders):
''' Gets an order corresponding to the ynab transaction '''
utils.log_debug('get_order', t, orders)
possible_orders = []
for order in orders.values():
# need negative because YNAB outflows have negative amount
if utils.equalish(order.total_charged, -afy.ynab.get_amount(t)):
possible_orders.append(order)
utils.log_debug('possible_orders', possible_orders)
if len(possible_orders) == 0:
utils.log_debug('No matching order for transaction')
return None
if len(possible_orders) == 1:
order = possible_orders[0]
else:
if afy.settings.fail_on_ambiguous_transaction:
utils.log_error('Skipping ambiguous transaction', t,
possible_orders)
return None
else:
utils.log_debug('Skipping ambiguous transaction', t,
possible_orders)
unused_orders = [
o for o in possible_orders if o.id not in assigned_ids
]
if not unused_orders:
return None
unused_orders.sort(
key=lambda o: utils.day_delta(t.date, o.shipment_date))
order = unused_orders[0]
assigned_ids.add(order.id)
utils.log_info('Matched transaction with order', t, order)
return order
def get_eligible_transactions(transactions):
utils.log_debug('get_eligible_transactions')
predicates = newer_than, has_blank_or_WIP_memo, matches_account
eligible = utils.multi_filter(predicates, transactions)
utils.log_info(
'Found %s transactions to attempt to match with Amazon orders' %
len(eligible))
return utils.by(eligible, lambda t: t.id)
def update_ynab(self):
categories = {
g.category.name: g.category
for p in self.priorities for g in p.goals
}
utils.log_info("Updating %s categories" % len(categories))
ynab.api_client.update_categories(
categories) # TODO: queue via ynab.ynab
def install():
utils.log_info('Installing')
setup_chromedriver()
make_dirs()
setup_ynab_auth()
setup_ynab_budget_id()
settings.init()
utils.log_debug('Settings:', settings._s.settings)
utils.log_info('Installed!')
def setup_chromedriver():
if os.path.exists(settings.chromedriver_path):
return
utils.log_info('Installing Chromedriver')
downloadPath = ChromeDriverManager(
path=settings.chromedriver_dir).install()
shutil.move(downloadPath, settings.chromedriver_path)
shutil.rmtree(settings.chromedriver_dir + "/drivers")
assert os.path.exists(settings.chromedriver_path)
def match_all(transactions, orders):
utils.log_debug('match_all', len(transactions), len(orders))
orders_by_transaction_id = {}
for t_id, t in transactions.items():
order = get_order(t, orders)
if not order:
continue
orders_by_transaction_id[t_id] = order
utils.log_info('Found %s matches' % len(orders_by_transaction_id))
return orders_by_transaction_id
def do_delete_transactions():
utils.log_debug('do_delete_transactions', transaction_delete_queue)
if not transaction_delete_queue:
return
utils.log_info('Set deletion memo on %s transactions via YNAB REST API' % len(transaction_delete_queue))
for t in transaction_delete_queue.values():
t.memo = delete_key
api_client.update_transactions(transaction_delete_queue)
utils.log_info('delete %s transactions via YNAB webapp' % len(transaction_delete_queue))
gui_client.delete_transactions()
transaction_delete_queue.clear()
def delete_accounts(accounts):
utils.log_info('Deleting %s accounts via Web App' % len(accounts))
load_gui()
navlink_accounts = gui.get('navlink-accounts')
gui.scroll_to(navlink_accounts)
gui.click(navlink_accounts)
for a in accounts:
edit_account = gui.get_by_text('nav-account-name', a.name)
utils.log_debug(edit_account)
gui.scroll_to(edit_account)
gui.right_click(edit_account)
gui.get_by_text('button-red', ['Close Account', 'Delete', 'Delete Account'], partial=True).click()
def update_amazon_transactions():
utils.log_info('Matching Amazon orders to YNAB transactions')
potential_amazon_transactions = afy.amazon.amazon.get_eligible_transactions(Assistant.transactions)
orders_by_transaction_id = afy.amazon.match.match_all(potential_amazon_transactions, Assistant.orders)
for t_id, order in orders_by_transaction_id.items():
order = orders_by_transaction_id[t_id]
i = Assistant.items[order.id]
assert i
t = Assistant.transactions.get(t_id)
afy.amazon.amazon.annotate(t, order, i)
# afy.ynab.ynab.queue_update(t, Assistant.payees, Assistant.categories)
afy.ynab.ynab.queue_update(t)
utils.log_info(utils.separator)
def restore_defaults():
utils.log_info('Restoring default settings')
for p in [settings.chromedriver_path, settings.settings_path]:
if os.path.exists(p):
utils.log_debug('removing', p)
os.remove(p)
for p in settings.log_dir, settings.data_dir, settings.backup_dir:
if os.path.exists(p):
utils.log_debug('removing', p)
shutil.rmtree(p)
settings.init()
def load_before(t, timestamp=settings.start_time):
''' Load the newest file made before timestamp '''
assert isinstance(timestamp, datetime.datetime)
paths = glob.glob(settings.backup_dir + '/*-' + str(t) + '-*.jsonpickle')
for path in sorted(paths, reverse=True):
filename = path.replace(settings.backup_dir + '/', '')
file_timestamp = filename[:filename.index('-<')]
if file_timestamp < str(timestamp):
break
else:
assert False
# return []
utils.log_info('Reloading from %s' % file_timestamp)
return load_path(path)
def add_adjustment_subtransaction(t):
''' Ensures that the sum of subtransaction prices equals the transaction amount '''
utils.log_debug('add_adjustment_subtransaction', t)
if not t.subtransactions:
return
amount = calculate_adjustment(t)
if not amount:
return
adjustment = utils.copy(t.subtransactions[0])
adjustment.memo = 'Split transaction adjustment'
adjustment.amount = amount
adjustment.category_name = afy.settings.default_category # TODO
utils.log_info('Warning, adjusting: subtransactions do not add up, by $%s' % -get_amount(adjustment))
t.subtransactions.append(adjustment)
assert utils.equalish(t.amount, sum(s.amount for s in t.subtransactions))
def annotate(t, order, items):
utils.log_debug('annotate', t, order, items)
t.date = order.order_date
if len(items) == 1:
annotate_with_item(t, items[0])
t.memo += ' ' + order.id
else:
t.memo = order.id
t.subtransactions = [
ynab_api.SubTransaction(
local_vars_configuration=afy.ynab.ynab.no_check_configuration)
for i in items
]
for i, s in zip(items, t.subtransactions):
annotate_with_item(s, i)
assert len(t.subtransactions) == len(items)
utils.log_info(t)
def enter_all_transactions(transactions):
utils.log_debug('enter_all_transactions', len(transactions))
load_gui()
for t in transactions:
utils.log_info(t)
if len(t.subtransactions) > 5:
utils.log_info(
'''Skipping purchase with %s items for speed reasons during alpha test.
Feel free to remove this check.''' % len(t.subtransactions))
continue
try:
enter_transaction(t)
except BaseException:
' Likely because there were multiple search results '
utils.log_exception()
utils.log_error('Error on transaction', t)
search = gui.get('transaction-search-input')
search.clear()
gui.quit()
def budget2(self):
unimportant = list(reversed(self.priorities))
for p1, p2 in zip(unimportant, unimportant[1:]):
before = p1.total_available() + p2.total_available()
amount = p1.total_available()
utils.log_info('moving', amount)
utils.log_info(p1, p2)
p1.distribute(-amount)
p2.distribute(amount)
utils.log_info('moved')
utils.log_info(p1, p2)
after = p1.total_available() + p2.total_available()
assert utils.equalish(after, before, -1)
def do_rest():
utils.log_debug('do_rest', rest_queue)
if not rest_queue:
return
for mode, ts in rest_queue.items():
utils.log_info('%s %s transactions via YNAB REST API' % (mode, len(ts)))
utils.log_debug(mode, ts)
'''
copied = copy.deepcopy(ts) # TODO: do we need copy?
for t in copied: # TODO: surely we don't need this
if t.subtransactions:
t.subtransactions = []
t.category_id = None
t.category_name = None
rest_modes[mode](copied)
'''
rest_modes[mode](ts)
utils.log_info(utils.separator)
rest_queue.clear()
def do_gui():
utils.log_debug('do_gui', gui_queue)
if not gui_queue:
return
old_memos = []
for mode, ts in gui_queue.items():
utils.log_info('%s %s transactions via YNAB webapp' % (mode, len(ts)))
utils.log_debug(mode, ts)
for t in ts.values(): # TODO: can this be simplified?
if len(t.subtransactions) <= 1:
utils.log_debug('Warning: no good reason to update via gui with %s subtransaction(s)' %
len(t.subtransactions), t)
# Ensures that we can find it in the gui
old_memos.append(annotate_for_locating(t))
rest_modes[mode](ts)
for m, t in zip(old_memos, ts.values()): # simplify out the .values? listy?
t.memo = m
add_adjustment_subtransaction(t)
gui_client.enter_all_transactions(ts)
utils.log_info(utils.separator)
gui_queue.clear()
gui.quit()
def setup_chromedriver():
if os.path.exists(settings.chromedriver_path):
return
utils.log_info('Installing Chromedriver')
chromedriver_urls = {
'windows':
'https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_win32.zip',
'mac':
'https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_mac64.zip',
'linux':
'https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_linux64.zip'
}
chromedriver_url = chromedriver_urls[settings.operating_system]
chromedriver_zip_filename = os.path.basename(chromedriver_url)
response = requests.get(chromedriver_url)
with open(chromedriver_zip_filename, 'wb') as f:
f.write(response.content)
with zipfile.ZipFile(chromedriver_zip_filename, 'r') as f:
f.extractall(settings.chromedriver_dir)
os.remove(chromedriver_zip_filename)
assert os.path.exists(settings.chromedriver_path)
def setup_ynab_auth():
utils.log_info('Checking for YNAB authentication')
# TODO: make this use oauth instead of api tokens
if settings.get('api_token'):
return
utils.log_info('Installing YNAB authentication')
api_token_url = 'https://app.youneedabudget.com/settings/developer'
d = gui.driver()
d.get(api_token_url)
utils.log_info('Log in if needed')
new_token_button = gui.get_by_text('button', 'New Token')
gui.click(new_token_button)
utils.log_info(
'Enter your password in the YNAB Web App, then click "Generate"')
while 'New Personal Access Token' not in d.page_source:
time.sleep(.5)
api_token = re.search(
'New Personal Access Token: <strong>([^<]*)</strong>',
d.page_source).groups()[0]
settings.set('api_token', api_token)
utils.log_debug('settings.api_token', settings.api_token)
assert settings.api_token
gui.quit()
def setup_ynab_budget_id():
utils.log_info('Checking for selected budget')
if settings.get('budget_id'):
return
utils.log_info('Selecting budget')
url = 'https://app.youneedabudget.com/'
driver = gui.driver()
driver.get(url)
utils.log_info('Log in if needed')
while not re.search('youneedabudget.com/([^/]+)/', driver.current_url):
time.sleep(.5)
budget_selection_prompt = 'Press Enter when you have loaded the budget you want to use.'
input(budget_selection_prompt)
utils.log_debug(budget_selection_prompt)
while not re.search('youneedabudget.com/([^/]+)/', driver.current_url):
time.sleep(.5)
settings.set(
'budget_id',
re.search('youneedabudget.com/([^/]+)/',
driver.current_url).groups()[0])
utils.log_debug('settings.budget_id', settings.budget_id)
assert settings.budget_id
gui.quit()
def do_confirm(modified, deleted, added):
utils.log_info("Nah, we won't confirm")
return True
def download_ynab(accounts=False, transactions=False, categories=False, payees=False):
utils.log_info('Downloading YNAB')
Assistant.load_ynab(accounts, transactions, categories, payees, local=False)
def restore_ynab(accounts=False, transactions=False, categories=False, payees=False):
utils.log_info('Restoring YNAB')
Assistant.load_ynab(accounts, transactions, categories, payees, local=True)
def make_dirs():
utils.log_info('Checking for directories')
for p in settings.log_dir, settings.data_dir, settings.backup_dir:
if not os.path.exists(p):
utils.log_info('Making directory', p)
os.mkdir(p)
def get_height():
utils.log_info('height', window_size()['height'])
return window_size()['height']
def load_ynab(accounts=False, transactions=False, categories=False, payees=False, local=False):
assert accounts or transactions or categories or payees
source = afy.backup.local if local else afy.ynab.api_client
# Need accounts to validate transactions pending deleted account bug fix
if transactions or accounts:
Assistant.accounts.store(source.get_accounts())
if accounts:
utils.log_info('Found %s accounts' % len(Assistant.accounts))
if transactions:
Assistant.transactions.store(source.get_transactions())
utils.log_info('Found %s transactions' % len(Assistant.transactions))
if categories:
Assistant.category_groups.store(source.get_category_groups())
utils.log_info('Found %s category groups' % len(Assistant.category_groups))
Assistant.categories.store(c for g in Assistant.category_groups for c in g.categories)
utils.log_info('Found %s categories' % len(Assistant.categories))
if payees:
Assistant.payees.store(source.get_payees())
utils.log_info('Found %s payees' % len(Assistant.payees))
utils.log_info(utils.separator)
def budget(self):
''' Fund high priority categories by withdrawing from low priority categories '''
important = iter(self.priorities)
unimportant = reversed(self.priorities)
sink = next(important)
source = next(unimportant)
# utils.debug()
while source is not sink:
utils.log_info(source, sink)
need = sink.total_need()
utils.log_info('need', need)
if utils.equalish(need, 0):
sink = next(important)
utils.log_info('new sink', sink)
available = source.total_available()
utils.log_info('available', available)
if available <= 0:
source = next(unimportant)
utils.log_info('new source', source)
use = min(available, need)
utils.log_info('use', use)
before = source.total_available() + sink.total_available()
utils.log_info('before', before)
utils.log_info('source before distribute', source)
source.distribute(-use)
utils.log_info('source after distribute', source)
utils.log_info('sink before distribute', sink)
sink.distribute(use)
utils.log_info('sink after distribute', sink)
after = source.total_available() + sink.total_available()
utils.log_info('after', after)
assert utils.equalish(after, before, -1)
