def diff_with_backup(predicates, timestamp=afy.settings.start_time, order='first'): all_backup_transactions = afy.backup.local.load_before(ynab_api.TransactionDetail, timestamp) matching = utils.multi_filter(predicates, all_backup_transactions) def key(t): return t.id # Get the first copy of the transaction saved # So that eg default arguments restore state at start of prior run unique = utils.by(get_unique(matching, key, order), key) unique_keys = set(unique.keys()) afy.Assistant.download_ynab(transactions=True) # TODO: can I get away with this? current_transactions = utils.by( utils.multi_filter( predicates, afy.Assistant.transactions.values()), key) current_keys = set(current_transactions.keys()) modified = unique_keys.intersection(current_keys) deleted = unique_keys - current_keys added = current_keys - unique_keys utils.log_info( 'Found %s modified, %s deleted, %s added transactions' % (len(modified), len(deleted), len(added))) return [list(filter(lambda t: t.id in x, matching)) for x in (modified, deleted, added)]
def driver(): utils.log_debug('driver') global _driver install.setup_chromedriver() try: if is_alive(_driver): return _driver close_orphan_drivers() options = Options() assert os.path.exists(settings.chrome_data_dir) options.add_argument('user-data-dir={}'.format( settings.chrome_data_dir)) options.add_argument('--disable-extensions') assert os.path.exists(settings.chromedriver_path) _driver = webdriver.Chrome(options=options, executable_path=settings.chromedriver_path) except BaseException: quit() if 'data directory is already in use' in traceback.format_exc(): utils.log_exception_debug() utils.log_error('Must close Selenium-controlled Chrome.') if input('Try again? [Y/n]\n').lower() != 'n': utils.log_info('Trying again') return driver() sys.exit() else: utils.log_exception() assert _driver return _driver
def load(data_type): utils.log_debug('load', data_type) assert data_type in data_parsers target_path = csv_paths[data_type] try: if not os.path.exists(target_path) or stale(target_path): d = gui.driver() url = 'https://smile.amazon.com/gp/b2b/reports' if url not in d.current_url: d.get(url) d.find_element_by_id('report-use-today').click() enter_start_date() d.find_element_by_id('report-type').click() d.find_element_by_id('report-type').send_keys(data_type) d.find_element_by_id('report-confirm').click() path = wait_for_download() utils.log_debug(path, target_path) os.rename(path, target_path) except BaseException: utils.log_exception_debug() if 'AttributeError' in traceback.format_exc(): # TODO: only retry on missing csv utils.log_info('real error', traceback.format_exc()) sys.exit() if input('One more try? [Y/n]').lower() != 'n': load(data_type) list_of_dicts = read(target_path) utils.log_info('Found %s %s' % (len(list_of_dicts), data_type)) return data_parsers[data_type](list_of_dicts)
def get_order(t, orders): ''' Gets an order corresponding to the ynab transaction ''' utils.log_debug('get_order', t, orders) possible_orders = [] for order in orders.values(): # need negative because YNAB outflows have negative amount if utils.equalish(order.total_charged, -afy.ynab.get_amount(t)): possible_orders.append(order) utils.log_debug('possible_orders', possible_orders) if len(possible_orders) == 0: utils.log_debug('No matching order for transaction') return None if len(possible_orders) == 1: order = possible_orders[0] else: if afy.settings.fail_on_ambiguous_transaction: utils.log_error('Skipping ambiguous transaction', t, possible_orders) return None else: utils.log_debug('Skipping ambiguous transaction', t, possible_orders) unused_orders = [ o for o in possible_orders if o.id not in assigned_ids ] if not unused_orders: return None unused_orders.sort( key=lambda o: utils.day_delta(t.date, o.shipment_date)) order = unused_orders[0] assigned_ids.add(order.id) utils.log_info('Matched transaction with order', t, order) return order
def get_eligible_transactions(transactions): utils.log_debug('get_eligible_transactions') predicates = newer_than, has_blank_or_WIP_memo, matches_account eligible = utils.multi_filter(predicates, transactions) utils.log_info( 'Found %s transactions to attempt to match with Amazon orders' % len(eligible)) return utils.by(eligible, lambda t: t.id)
def update_ynab(self): categories = { g.category.name: g.category for p in self.priorities for g in p.goals } utils.log_info("Updating %s categories" % len(categories)) ynab.api_client.update_categories( categories) # TODO: queue via ynab.ynab
def install(): utils.log_info('Installing') setup_chromedriver() make_dirs() setup_ynab_auth() setup_ynab_budget_id() settings.init() utils.log_debug('Settings:', settings._s.settings) utils.log_info('Installed!')
def setup_chromedriver(): if os.path.exists(settings.chromedriver_path): return utils.log_info('Installing Chromedriver') downloadPath = ChromeDriverManager( path=settings.chromedriver_dir).install() shutil.move(downloadPath, settings.chromedriver_path) shutil.rmtree(settings.chromedriver_dir + "/drivers") assert os.path.exists(settings.chromedriver_path)
def match_all(transactions, orders): utils.log_debug('match_all', len(transactions), len(orders)) orders_by_transaction_id = {} for t_id, t in transactions.items(): order = get_order(t, orders) if not order: continue orders_by_transaction_id[t_id] = order utils.log_info('Found %s matches' % len(orders_by_transaction_id)) return orders_by_transaction_id
def do_delete_transactions(): utils.log_debug('do_delete_transactions', transaction_delete_queue) if not transaction_delete_queue: return utils.log_info('Set deletion memo on %s transactions via YNAB REST API' % len(transaction_delete_queue)) for t in transaction_delete_queue.values(): t.memo = delete_key api_client.update_transactions(transaction_delete_queue) utils.log_info('delete %s transactions via YNAB webapp' % len(transaction_delete_queue)) gui_client.delete_transactions() transaction_delete_queue.clear()
def delete_accounts(accounts): utils.log_info('Deleting %s accounts via Web App' % len(accounts)) load_gui() navlink_accounts = gui.get('navlink-accounts') gui.scroll_to(navlink_accounts) gui.click(navlink_accounts) for a in accounts: edit_account = gui.get_by_text('nav-account-name', a.name) utils.log_debug(edit_account) gui.scroll_to(edit_account) gui.right_click(edit_account) gui.get_by_text('button-red', ['Close Account', 'Delete', 'Delete Account'], partial=True).click()
def update_amazon_transactions(): utils.log_info('Matching Amazon orders to YNAB transactions') potential_amazon_transactions = afy.amazon.amazon.get_eligible_transactions(Assistant.transactions) orders_by_transaction_id = afy.amazon.match.match_all(potential_amazon_transactions, Assistant.orders) for t_id, order in orders_by_transaction_id.items(): order = orders_by_transaction_id[t_id] i = Assistant.items[order.id] assert i t = Assistant.transactions.get(t_id) afy.amazon.amazon.annotate(t, order, i) # afy.ynab.ynab.queue_update(t, Assistant.payees, Assistant.categories) afy.ynab.ynab.queue_update(t) utils.log_info(utils.separator)
def restore_defaults(): utils.log_info('Restoring default settings') for p in [settings.chromedriver_path, settings.settings_path]: if os.path.exists(p): utils.log_debug('removing', p) os.remove(p) for p in settings.log_dir, settings.data_dir, settings.backup_dir: if os.path.exists(p): utils.log_debug('removing', p) shutil.rmtree(p) settings.init()
def load_before(t, timestamp=settings.start_time): ''' Load the newest file made before timestamp ''' assert isinstance(timestamp, datetime.datetime) paths = glob.glob(settings.backup_dir + '/*-' + str(t) + '-*.jsonpickle') for path in sorted(paths, reverse=True): filename = path.replace(settings.backup_dir + '/', '') file_timestamp = filename[:filename.index('-<')] if file_timestamp < str(timestamp): break else: assert False # return [] utils.log_info('Reloading from %s' % file_timestamp) return load_path(path)
def add_adjustment_subtransaction(t): ''' Ensures that the sum of subtransaction prices equals the transaction amount ''' utils.log_debug('add_adjustment_subtransaction', t) if not t.subtransactions: return amount = calculate_adjustment(t) if not amount: return adjustment = utils.copy(t.subtransactions[0]) adjustment.memo = 'Split transaction adjustment' adjustment.amount = amount adjustment.category_name = afy.settings.default_category # TODO utils.log_info('Warning, adjusting: subtransactions do not add up, by $%s' % -get_amount(adjustment)) t.subtransactions.append(adjustment) assert utils.equalish(t.amount, sum(s.amount for s in t.subtransactions))
def annotate(t, order, items): utils.log_debug('annotate', t, order, items) t.date = order.order_date if len(items) == 1: annotate_with_item(t, items[0]) t.memo += ' ' + order.id else: t.memo = order.id t.subtransactions = [ ynab_api.SubTransaction( local_vars_configuration=afy.ynab.ynab.no_check_configuration) for i in items ] for i, s in zip(items, t.subtransactions): annotate_with_item(s, i) assert len(t.subtransactions) == len(items) utils.log_info(t)
def enter_all_transactions(transactions): utils.log_debug('enter_all_transactions', len(transactions)) load_gui() for t in transactions: utils.log_info(t) if len(t.subtransactions) > 5: utils.log_info( '''Skipping purchase with %s items for speed reasons during alpha test. Feel free to remove this check.''' % len(t.subtransactions)) continue try: enter_transaction(t) except BaseException: ' Likely because there were multiple search results ' utils.log_exception() utils.log_error('Error on transaction', t) search = gui.get('transaction-search-input') search.clear() gui.quit()
def budget2(self): unimportant = list(reversed(self.priorities)) for p1, p2 in zip(unimportant, unimportant[1:]): before = p1.total_available() + p2.total_available() amount = p1.total_available() utils.log_info('moving', amount) utils.log_info(p1, p2) p1.distribute(-amount) p2.distribute(amount) utils.log_info('moved') utils.log_info(p1, p2) after = p1.total_available() + p2.total_available() assert utils.equalish(after, before, -1)
def do_rest(): utils.log_debug('do_rest', rest_queue) if not rest_queue: return for mode, ts in rest_queue.items(): utils.log_info('%s %s transactions via YNAB REST API' % (mode, len(ts))) utils.log_debug(mode, ts) ''' copied = copy.deepcopy(ts) # TODO: do we need copy? for t in copied: # TODO: surely we don't need this if t.subtransactions: t.subtransactions = [] t.category_id = None t.category_name = None rest_modes[mode](copied) ''' rest_modes[mode](ts) utils.log_info(utils.separator) rest_queue.clear()
def do_gui(): utils.log_debug('do_gui', gui_queue) if not gui_queue: return old_memos = [] for mode, ts in gui_queue.items(): utils.log_info('%s %s transactions via YNAB webapp' % (mode, len(ts))) utils.log_debug(mode, ts) for t in ts.values(): # TODO: can this be simplified? if len(t.subtransactions) <= 1: utils.log_debug('Warning: no good reason to update via gui with %s subtransaction(s)' % len(t.subtransactions), t) # Ensures that we can find it in the gui old_memos.append(annotate_for_locating(t)) rest_modes[mode](ts) for m, t in zip(old_memos, ts.values()): # simplify out the .values? listy? t.memo = m add_adjustment_subtransaction(t) gui_client.enter_all_transactions(ts) utils.log_info(utils.separator) gui_queue.clear() gui.quit()
def setup_chromedriver(): if os.path.exists(settings.chromedriver_path): return utils.log_info('Installing Chromedriver') chromedriver_urls = { 'windows': 'https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_win32.zip', 'mac': 'https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_mac64.zip', 'linux': 'https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_linux64.zip' } chromedriver_url = chromedriver_urls[settings.operating_system] chromedriver_zip_filename = os.path.basename(chromedriver_url) response = requests.get(chromedriver_url) with open(chromedriver_zip_filename, 'wb') as f: f.write(response.content) with zipfile.ZipFile(chromedriver_zip_filename, 'r') as f: f.extractall(settings.chromedriver_dir) os.remove(chromedriver_zip_filename) assert os.path.exists(settings.chromedriver_path)
def setup_ynab_auth(): utils.log_info('Checking for YNAB authentication') # TODO: make this use oauth instead of api tokens if settings.get('api_token'): return utils.log_info('Installing YNAB authentication') api_token_url = 'https://app.youneedabudget.com/settings/developer' d = gui.driver() d.get(api_token_url) utils.log_info('Log in if needed') new_token_button = gui.get_by_text('button', 'New Token') gui.click(new_token_button) utils.log_info( 'Enter your password in the YNAB Web App, then click "Generate"') while 'New Personal Access Token' not in d.page_source: time.sleep(.5) api_token = re.search( 'New Personal Access Token: <strong>([^<]*)</strong>', d.page_source).groups()[0] settings.set('api_token', api_token) utils.log_debug('settings.api_token', settings.api_token) assert settings.api_token gui.quit()
def setup_ynab_budget_id(): utils.log_info('Checking for selected budget') if settings.get('budget_id'): return utils.log_info('Selecting budget') url = 'https://app.youneedabudget.com/' driver = gui.driver() driver.get(url) utils.log_info('Log in if needed') while not re.search('youneedabudget.com/([^/]+)/', driver.current_url): time.sleep(.5) budget_selection_prompt = 'Press Enter when you have loaded the budget you want to use.' input(budget_selection_prompt) utils.log_debug(budget_selection_prompt) while not re.search('youneedabudget.com/([^/]+)/', driver.current_url): time.sleep(.5) settings.set( 'budget_id', re.search('youneedabudget.com/([^/]+)/', driver.current_url).groups()[0]) utils.log_debug('settings.budget_id', settings.budget_id) assert settings.budget_id gui.quit()
def do_confirm(modified, deleted, added): utils.log_info("Nah, we won't confirm") return True
def download_ynab(accounts=False, transactions=False, categories=False, payees=False): utils.log_info('Downloading YNAB') Assistant.load_ynab(accounts, transactions, categories, payees, local=False)
def restore_ynab(accounts=False, transactions=False, categories=False, payees=False): utils.log_info('Restoring YNAB') Assistant.load_ynab(accounts, transactions, categories, payees, local=True)
def make_dirs(): utils.log_info('Checking for directories') for p in settings.log_dir, settings.data_dir, settings.backup_dir: if not os.path.exists(p): utils.log_info('Making directory', p) os.mkdir(p)
def get_height(): utils.log_info('height', window_size()['height']) return window_size()['height']
def load_ynab(accounts=False, transactions=False, categories=False, payees=False, local=False): assert accounts or transactions or categories or payees source = afy.backup.local if local else afy.ynab.api_client # Need accounts to validate transactions pending deleted account bug fix if transactions or accounts: Assistant.accounts.store(source.get_accounts()) if accounts: utils.log_info('Found %s accounts' % len(Assistant.accounts)) if transactions: Assistant.transactions.store(source.get_transactions()) utils.log_info('Found %s transactions' % len(Assistant.transactions)) if categories: Assistant.category_groups.store(source.get_category_groups()) utils.log_info('Found %s category groups' % len(Assistant.category_groups)) Assistant.categories.store(c for g in Assistant.category_groups for c in g.categories) utils.log_info('Found %s categories' % len(Assistant.categories)) if payees: Assistant.payees.store(source.get_payees()) utils.log_info('Found %s payees' % len(Assistant.payees)) utils.log_info(utils.separator)
def budget(self): ''' Fund high priority categories by withdrawing from low priority categories ''' important = iter(self.priorities) unimportant = reversed(self.priorities) sink = next(important) source = next(unimportant) # utils.debug() while source is not sink: utils.log_info(source, sink) need = sink.total_need() utils.log_info('need', need) if utils.equalish(need, 0): sink = next(important) utils.log_info('new sink', sink) available = source.total_available() utils.log_info('available', available) if available <= 0: source = next(unimportant) utils.log_info('new source', source) use = min(available, need) utils.log_info('use', use) before = source.total_available() + sink.total_available() utils.log_info('before', before) utils.log_info('source before distribute', source) source.distribute(-use) utils.log_info('source after distribute', source) utils.log_info('sink before distribute', sink) sink.distribute(use) utils.log_info('sink after distribute', sink) after = source.total_available() + sink.total_available() utils.log_info('after', after) assert utils.equalish(after, before, -1)