def do_create_updates(self, args, parent): def on_mint_mfa(prompt): logger.info('Asking for Mint MFA') self.on_mint_mfa.emit() logger.info('Blocking') self.mfa_condition.wait() logger.info('got code!') logger.info(self.mfa_code) return self.mfa_code # Factory that handles indeterminite, determinite, and counter style. def progress_factory(msg, max=0): return QtProgress(msg, max, self.on_progress.emit) self.mint_client = MintClient(email=args.mint_email, password=args.mint_password, session_path=args.session_path, headless=args.headless, mfa_method=args.mint_mfa_method, wait_for_sync=args.mint_wait_for_sync, mfa_input_callback=on_mint_mfa, progress_factory=progress_factory) results = tagger.create_updates( args, self.mint_client, on_critical=self.on_error.emit, indeterminate_progress_factory=progress_factory, determinate_progress_factory=progress_factory, counter_progress_factory=progress_factory) if results.success and not self.stopping: self.on_review_ready.emit(results)
def do_create_updates(self, args, parent): def on_mint_mfa(prompt): logger.info('Asking for Mint MFA') self.on_mint_mfa.emit() loop = QEventLoop() self.on_mint_mfa_done.connect(loop.quit) loop.exec_() logger.info(self.mfa_code) return self.mfa_code # Factory that handles indeterminite, determinite, and counter style. def progress_factory(msg, max=0): return QtProgress(msg, max, self.on_progress.emit) atexit.register(self.close_webdriver) bound_webdriver_factory = partial(self.get_webdriver, args) self.mint_client = MintClient(args, bound_webdriver_factory, mfa_input_callback=on_mint_mfa) if not fetch_order_history(args, bound_webdriver_factory, progress_factory): self.on_error.emit( 'Failed to fetch Amazon order history. Check credentials') return results = tagger.create_updates( args, self.mint_client, on_critical=self.on_error.emit, indeterminate_progress_factory=progress_factory, determinate_progress_factory=progress_factory, counter_progress_factory=progress_factory) if results.success and not self.stopping: self.on_review_ready.emit(results)
class TaggerWorker(QObject): """This class is required to prevent locking up the main Qt thread.""" on_error = pyqtSignal(str) on_review_ready = pyqtSignal(tagger.UpdatesResult) on_updates_sent = pyqtSignal(int) on_stopped = pyqtSignal() on_mint_mfa = pyqtSignal() on_progress = pyqtSignal(str, int, int) stopping = False mfa_condition = Condition() @pyqtSlot() def stop(self): self.stopping = True @pyqtSlot(str) def mfa_code(self, code): logger.info('Got code') logger.info(code) self.mfa_code = code logger.info('Waking thread') self.mfa_condition.notify() @pyqtSlot(object) def create_updates(self, args, parent): try: self.do_create_updates(args, parent) except Exception as e: msg = 'Internal error while creating updates: {}'.format(e) self.on_error.emit(msg) logger.exception(msg) @pyqtSlot(list, object) def send_updates(self, updates, args): try: self.do_send_updates(updates, args) except Exception as e: msg = 'Internal error while sending updates: {}'.format(e) self.on_error.emit(msg) logger.exception(msg) def do_create_updates(self, args, parent): def on_mint_mfa(prompt): logger.info('Asking for Mint MFA') self.on_mint_mfa.emit() logger.info('Blocking') self.mfa_condition.wait() logger.info('got code!') logger.info(self.mfa_code) return self.mfa_code # Factory that handles indeterminite, determinite, and counter style. def progress_factory(msg, max=0): return QtProgress(msg, max, self.on_progress.emit) self.mint_client = MintClient(email=args.mint_email, password=args.mint_password, session_path=args.session_path, headless=args.headless, mfa_method=args.mint_mfa_method, wait_for_sync=args.mint_wait_for_sync, mfa_input_callback=on_mint_mfa, progress_factory=progress_factory) results = tagger.create_updates( args, self.mint_client, on_critical=self.on_error.emit, indeterminate_progress_factory=progress_factory, determinate_progress_factory=progress_factory, counter_progress_factory=progress_factory) if results.success and not self.stopping: self.on_review_ready.emit(results) def do_send_updates(self, updates, args): num_updates = self.mint_client.send_updates( updates, progress=QtProgress('Sending updates to Mint', len(updates), self.on_progress.emit), ignore_category=args.no_tag_categories) self.on_updates_sent.emit(num_updates) self.mint_client.close()
def main(): warn_if_outdated('mint-amazon-tagger', VERSION) is_outdated, latest_version = check_outdated('mint-amazon-tagger', VERSION) if is_outdated: print('Please update your version by running:\n' 'pip3 install mint-amazon-tagger --upgrade') parser = argparse.ArgumentParser( description='Tag Mint transactions based on itemized Amazon history.') define_args(parser) args = parser.parse_args() if args.version: print('mint-amazon-tagger {}\nBy: Jeff Prouty'.format(VERSION)) exit(0) session_path = args.session_path if session_path.lower() == 'none': session_path = None items_csv = args.items_csv orders_csv = args.orders_csv refunds_csv = args.refunds_csv start_date = None if not items_csv or not orders_csv: logger.info('Missing Items/Orders History csv. Attempting to fetch ' 'from Amazon.com.') start_date = args.order_history_start_date duration = datetime.timedelta(days=args.order_history_num_days) end_date = datetime.date.today() # If a start date is given, adjust the end date based on num_days, # ensuring not to go beyond today. if start_date: start_date = start_date.date() if start_date + duration < end_date: end_date = start_date + duration else: start_date = end_date - duration items_csv, orders_csv, refunds_csv = fetch_order_history( args.report_download_location, start_date, end_date, args.amazon_email, args.amazon_password, session_path, args.headless) if not items_csv or not orders_csv: # Refunds are optional logger.critical('Order history either not provided at command line or ' 'unable to fetch. Exiting.') exit(1) orders = amazon.Order.parse_from_csv(orders_csv, ProgressCounter('Parsing Orders - ')) items = amazon.Item.parse_from_csv(items_csv, ProgressCounter('Parsing Items - ')) refunds = ([] if not refunds_csv else amazon.Refund.parse_from_csv( refunds_csv, ProgressCounter('Parsing Refunds - '))) if args.dry_run: logger.info('\nDry Run; no modifications being sent to Mint.\n') # Initialize the stats. Explicitly initialize stats that might not be # accumulated (conditionals). stats = Counter( adjust_itemized_tax=0, already_up_to_date=0, misc_charge=0, new_tag=0, no_retag=0, retag=0, user_skipped_retag=0, personal_cat=0, ) mint_client = MintClient(args.mint_email, args.mint_password, session_path, args.headless, args.mint_mfa_method, args.wait_for_sync) if args.pickled_epoch: mint_trans, mint_category_name_to_id = ( get_trans_and_categories_from_pickle(args.pickled_epoch, args.mint_pickle_location)) else: # Get the date of the oldest Amazon order. if not start_date: start_date = min([o.order_date for o in orders]) if refunds: start_date = min(start_date, min([o.order_date for o in refunds])) # Double the length of transaction history to help aid in # personalized category tagging overrides. today = datetime.date.today() start_date = today - (today - start_date) * 2 mint_category_name_to_id = mint_client.get_categories() mint_transactions_json = mint_client.get_transactions(start_date) epoch = int(time.time()) mint_trans = mint.Transaction.parse_from_json(mint_transactions_json) dump_trans_and_categories(mint_trans, mint_category_name_to_id, epoch, args.mint_pickle_location) updates, unmatched_orders = tagger.get_mint_updates( orders, items, refunds, mint_trans, args, stats, mint_category_name_to_id) log_amazon_stats(items, orders, refunds) log_processing_stats(stats) if args.print_unmatched and unmatched_orders: logger.warning( 'The following were not matched to Mint transactions:\n') by_oid = defaultdict(list) for uo in unmatched_orders: by_oid[uo.order_id].append(uo) for unmatched_by_oid in by_oid.values(): orders = [o for o in unmatched_by_oid if o.is_debit] refunds = [o for o in unmatched_by_oid if not o.is_debit] if orders: print_unmatched(amazon.Order.merge(orders)) for r in amazon.Refund.merge(refunds): print_unmatched(r) if not updates: logger.info( 'All done; no new tags to be updated at this point in time!') exit(0) if args.dry_run: logger.info('Dry run. Following are proposed changes:') if args.skip_dry_print: logger.info('Dry run print results skipped!') else: tagger.print_dry_run(updates, ignore_category=args.no_tag_categories) else: mint_client.send_updates(updates, ignore_category=args.no_tag_categories)
class TaggerWorker(QObject): """This class is required to prevent locking up the main Qt thread.""" on_error = pyqtSignal(str) on_review_ready = pyqtSignal(tagger.UpdatesResult) on_updates_sent = pyqtSignal(int) on_stopped = pyqtSignal() on_mint_mfa = pyqtSignal() on_mint_mfa_done = pyqtSignal() on_progress = pyqtSignal(str, int, int) stopping = False webdriver = None @pyqtSlot() def stop(self): self.stopping = True @pyqtSlot(str) def mfa_code(self, code): logger.info(code) self.mfa_code = code @pyqtSlot(object) def create_updates(self, args, parent): try: self.do_create_updates(args, parent) except Exception as e: msg = 'Internal error while creating updates: {}'.format(e) self.on_error.emit(msg) logger.exception(msg) @pyqtSlot(list, object) def send_updates(self, updates, args): try: self.do_send_updates(updates, args) except Exception as e: msg = 'Internal error while sending updates: {}'.format(e) self.on_error.emit(msg) logger.exception(msg) def close_webdriver(self): if self.webdriver: self.webdriver.close() self.webdriver = None def get_webdriver(self, args): if self.webdriver: logger.info('Using existing webdriver') return self.webdriver logger.info('Creating a new webdriver') self.webdriver = get_webdriver(args.headless, args.session_path) return self.webdriver def do_create_updates(self, args, parent): def on_mint_mfa(prompt): logger.info('Asking for Mint MFA') self.on_mint_mfa.emit() loop = QEventLoop() self.on_mint_mfa_done.connect(loop.quit) loop.exec_() logger.info(self.mfa_code) return self.mfa_code # Factory that handles indeterminite, determinite, and counter style. def progress_factory(msg, max=0): return QtProgress(msg, max, self.on_progress.emit) atexit.register(self.close_webdriver) bound_webdriver_factory = partial(self.get_webdriver, args) self.mint_client = MintClient(args, bound_webdriver_factory, mfa_input_callback=on_mint_mfa) if not fetch_order_history(args, bound_webdriver_factory, progress_factory): self.on_error.emit( 'Failed to fetch Amazon order history. Check credentials') return results = tagger.create_updates( args, self.mint_client, on_critical=self.on_error.emit, indeterminate_progress_factory=progress_factory, determinate_progress_factory=progress_factory, counter_progress_factory=progress_factory) if results.success and not self.stopping: self.on_review_ready.emit(results) def do_send_updates(self, updates, args): num_updates = self.mint_client.send_updates( updates, progress=QtProgress('Sending updates to Mint', len(updates), self.on_progress.emit), ignore_category=args.no_tag_categories) self.close_webdriver() self.on_updates_sent.emit(num_updates)
def main(): root_logger = logging.getLogger() root_logger.addHandler(logging.StreamHandler()) # For helping remote debugging, also log to file. # Developers should be vigilant to NOT log any PII, ever (including being # mindful of what exceptions might be thrown). log_directory = os.path.join(TAGGER_BASE_PATH, 'Tagger Logs') os.makedirs(log_directory, exist_ok=True) log_filename = os.path.join( log_directory, '{}.log'.format(time.strftime("%Y-%m-%d_%H-%M-%S"))) root_logger.addHandler(logging.FileHandler(log_filename)) is_outdated, latest_version = check_outdated('mint-amazon-tagger', VERSION) if is_outdated: print('Please update your version by running:\n' 'pip3 install mint-amazon-tagger --upgrade\n\n') parser = argparse.ArgumentParser( description='Tag Mint transactions based on itemized Amazon history.') define_cli_args(parser) args = parser.parse_args() if args.version: print('mint-amazon-tagger {}\nBy: Jeff Prouty'.format(VERSION)) exit(0) mint_client = MintClient(email=args.mint_email, password=args.mint_password, session_path=args.session_path, headless=args.headless, mfa_method=args.mint_mfa_method, wait_for_sync=args.mint_wait_for_sync, progress_factory=indeterminate_progress_cli) if args.dry_run: logger.info('\nDry Run; no modifications being sent to Mint.\n') def on_critical(msg): logger.critical(msg) exit(1) results = tagger.create_updates( args, mint_client, on_critical=on_critical, indeterminate_progress_factory=indeterminate_progress_cli, determinate_progress_factory=determinate_progress_cli, counter_progress_factory=counter_progress_cli) if not results.success: logger.critical('Uncaught error from create_updates. Exiting') exit(1) log_amazon_stats(results.items, results.orders, results.refunds) log_processing_stats(results.stats) if args.print_unmatched and results.unmatched_orders: logger.warning( 'The following were not matched to Mint transactions:\n') by_oid = defaultdict(list) for uo in results.unmatched_orders: by_oid[uo.order_id].append(uo) for unmatched_by_oid in by_oid.values(): orders = [o for o in unmatched_by_oid if o.is_debit] refunds = [o for o in unmatched_by_oid if not o.is_debit] if orders: print_unmatched(amazon.Order.merge(orders)) for r in amazon.Refund.merge(refunds): print_unmatched(r) if not results.updates: logger.info( 'All done; no new tags to be updated at this point in time!') exit(0) if args.dry_run: logger.info('Dry run. Following are proposed changes:') if args.skip_dry_print: logger.info('Dry run print results skipped!') else: tagger.print_dry_run(results.updates, ignore_category=args.no_tag_categories) else: num_updates = mint_client.send_updates( results.updates, progress=determinate_progress_cli('Updating Mint', max=len(results.updates)), ignore_category=args.no_tag_categories) logger.info('Sent {} updates to Mint'.format(num_updates))
def main(): root_logger = logging.getLogger() root_logger.setLevel(logging.INFO) root_logger.addHandler(logging.StreamHandler()) # Disable noisy log spam from filelock from within tldextract. logging.getLogger("filelock").setLevel(logging.WARN) # For helping remote debugging, also log to file. # Developers should be vigilant to NOT log any PII, ever (including being # mindful of what exceptions might be thrown). log_directory = os.path.join(TAGGER_BASE_PATH, 'Tagger Logs') os.makedirs(log_directory, exist_ok=True) log_filename = os.path.join(log_directory, '{}.log'.format( time.strftime("%Y-%m-%d_%H-%M-%S"))) root_logger.addHandler(logging.FileHandler(log_filename)) is_outdated, latest_version = check_outdated('mint-amazon-tagger', VERSION) if is_outdated: print('Please update your version by running:\n' 'pip3 install mint-amazon-tagger --upgrade\n\n') parser = argparse.ArgumentParser( description='Tag Mint transactions based on itemized Amazon history.') define_cli_args(parser) args = parser.parse_args() if args.version: print('mint-amazon-tagger {}\nBy: Jeff Prouty'.format(VERSION)) exit(0) webdriver = None def close_webdriver(): if webdriver: webdriver.close() atexit.register(close_webdriver) def webdriver_factory(): nonlocal webdriver if webdriver: return webdriver webdriver = get_webdriver(args.headless, args.session_path) return webdriver mint_client = MintClient(args, webdriver_factory) # Attempt to fetch the order history if csv files are not already provided. if not has_order_history_csv_files(args): if not maybe_prompt_for_amazon_credentials(args): logger.critical('Failed to get Amazon credentials.') exit(1) if not fetch_order_history( args, webdriver_factory, indeterminate_progress_cli): logger.critical('Failed to fetch Amazon order history.') exit(1) if args.dry_run: logger.info('\nDry Run; no modifications being sent to Mint.\n') def on_critical(msg): logger.critical(msg) exit(1) maybe_prompt_for_mint_credentials(args) results = tagger.create_updates( args, mint_client, on_critical=on_critical, indeterminate_progress_factory=indeterminate_progress_cli, determinate_progress_factory=determinate_progress_cli, counter_progress_factory=counter_progress_cli) if not results.success: logger.critical('Uncaught error from create_updates. Exiting') exit(1) log_amazon_stats(results.items, results.orders, results.refunds) log_processing_stats(results.stats) if args.print_unmatched and results.unmatched_orders: logger.warning( 'The following were not matched to Mint transactions:\n') by_oid = defaultdict(list) for uo in results.unmatched_orders: by_oid[uo.order_id].append(uo) for unmatched_by_oid in by_oid.values(): orders = [o for o in unmatched_by_oid if o.is_debit] refunds = [o for o in unmatched_by_oid if not o.is_debit] if orders: print_unmatched(amazon.Order.merge(orders)) for r in amazon.Refund.merge(refunds): print_unmatched(r) if not results.updates: logger.info( 'All done; no new tags to be updated at this point in time!') exit(0) if args.dry_run: logger.info('Dry run. Following are proposed changes:') if args.skip_dry_print: logger.info('Dry run print results skipped!') else: tagger.print_dry_run(results.updates, ignore_category=args.no_tag_categories) else: num_updates = mint_client.send_updates( results.updates, progress=determinate_progress_cli( 'Updating Mint', max=len(results.updates)), ignore_category=args.no_tag_categories) logger.info('Sent {} updates to Mint'.format(num_updates))