def do_create_updates(self, args, parent):
def on_mint_mfa(prompt):
logger.info('Asking for Mint MFA')
self.on_mint_mfa.emit()
logger.info('Blocking')
self.mfa_condition.wait()
logger.info('got code!')
logger.info(self.mfa_code)
return self.mfa_code
# Factory that handles indeterminite, determinite, and counter style.
def progress_factory(msg, max=0):
return QtProgress(msg, max, self.on_progress.emit)
self.mint_client = MintClient(email=args.mint_email,
password=args.mint_password,
session_path=args.session_path,
headless=args.headless,
mfa_method=args.mint_mfa_method,
wait_for_sync=args.mint_wait_for_sync,
mfa_input_callback=on_mint_mfa,
progress_factory=progress_factory)
results = tagger.create_updates(
args,
self.mint_client,
on_critical=self.on_error.emit,
indeterminate_progress_factory=progress_factory,
determinate_progress_factory=progress_factory,
counter_progress_factory=progress_factory)
if results.success and not self.stopping:
self.on_review_ready.emit(results)
def do_create_updates(self, args, parent):
def on_mint_mfa(prompt):
logger.info('Asking for Mint MFA')
self.on_mint_mfa.emit()
loop = QEventLoop()
self.on_mint_mfa_done.connect(loop.quit)
loop.exec_()
logger.info(self.mfa_code)
return self.mfa_code
# Factory that handles indeterminite, determinite, and counter style.
def progress_factory(msg, max=0):
return QtProgress(msg, max, self.on_progress.emit)
atexit.register(self.close_webdriver)
bound_webdriver_factory = partial(self.get_webdriver, args)
self.mint_client = MintClient(args,
bound_webdriver_factory,
mfa_input_callback=on_mint_mfa)
if not fetch_order_history(args, bound_webdriver_factory,
progress_factory):
self.on_error.emit(
'Failed to fetch Amazon order history. Check credentials')
return
results = tagger.create_updates(
args,
self.mint_client,
on_critical=self.on_error.emit,
indeterminate_progress_factory=progress_factory,
determinate_progress_factory=progress_factory,
counter_progress_factory=progress_factory)
if results.success and not self.stopping:
self.on_review_ready.emit(results)
class TaggerWorker(QObject):
"""This class is required to prevent locking up the main Qt thread."""
on_error = pyqtSignal(str)
on_review_ready = pyqtSignal(tagger.UpdatesResult)
on_updates_sent = pyqtSignal(int)
on_stopped = pyqtSignal()
on_mint_mfa = pyqtSignal()
on_progress = pyqtSignal(str, int, int)
stopping = False
mfa_condition = Condition()
@pyqtSlot()
def stop(self):
self.stopping = True
@pyqtSlot(str)
def mfa_code(self, code):
logger.info('Got code')
logger.info(code)
self.mfa_code = code
logger.info('Waking thread')
self.mfa_condition.notify()
@pyqtSlot(object)
def create_updates(self, args, parent):
try:
self.do_create_updates(args, parent)
except Exception as e:
msg = 'Internal error while creating updates: {}'.format(e)
self.on_error.emit(msg)
logger.exception(msg)
@pyqtSlot(list, object)
def send_updates(self, updates, args):
try:
self.do_send_updates(updates, args)
except Exception as e:
msg = 'Internal error while sending updates: {}'.format(e)
self.on_error.emit(msg)
logger.exception(msg)
def do_create_updates(self, args, parent):
def on_mint_mfa(prompt):
logger.info('Asking for Mint MFA')
self.on_mint_mfa.emit()
logger.info('Blocking')
self.mfa_condition.wait()
logger.info('got code!')
logger.info(self.mfa_code)
return self.mfa_code
# Factory that handles indeterminite, determinite, and counter style.
def progress_factory(msg, max=0):
return QtProgress(msg, max, self.on_progress.emit)
self.mint_client = MintClient(email=args.mint_email,
password=args.mint_password,
session_path=args.session_path,
headless=args.headless,
mfa_method=args.mint_mfa_method,
wait_for_sync=args.mint_wait_for_sync,
mfa_input_callback=on_mint_mfa,
progress_factory=progress_factory)
results = tagger.create_updates(
args,
self.mint_client,
on_critical=self.on_error.emit,
indeterminate_progress_factory=progress_factory,
determinate_progress_factory=progress_factory,
counter_progress_factory=progress_factory)
if results.success and not self.stopping:
self.on_review_ready.emit(results)
def do_send_updates(self, updates, args):
num_updates = self.mint_client.send_updates(
updates,
progress=QtProgress('Sending updates to Mint', len(updates),
self.on_progress.emit),
ignore_category=args.no_tag_categories)
self.on_updates_sent.emit(num_updates)
self.mint_client.close()
def main():
warn_if_outdated('mint-amazon-tagger', VERSION)
is_outdated, latest_version = check_outdated('mint-amazon-tagger', VERSION)
if is_outdated:
print('Please update your version by running:\n'
'pip3 install mint-amazon-tagger --upgrade')
parser = argparse.ArgumentParser(
description='Tag Mint transactions based on itemized Amazon history.')
define_args(parser)
args = parser.parse_args()
if args.version:
print('mint-amazon-tagger {}\nBy: Jeff Prouty'.format(VERSION))
exit(0)
session_path = args.session_path
if session_path.lower() == 'none':
session_path = None
items_csv = args.items_csv
orders_csv = args.orders_csv
refunds_csv = args.refunds_csv
start_date = None
if not items_csv or not orders_csv:
logger.info('Missing Items/Orders History csv. Attempting to fetch '
'from Amazon.com.')
start_date = args.order_history_start_date
duration = datetime.timedelta(days=args.order_history_num_days)
end_date = datetime.date.today()
# If a start date is given, adjust the end date based on num_days,
# ensuring not to go beyond today.
if start_date:
start_date = start_date.date()
if start_date + duration < end_date:
end_date = start_date + duration
else:
start_date = end_date - duration
items_csv, orders_csv, refunds_csv = fetch_order_history(
args.report_download_location, start_date, end_date,
args.amazon_email, args.amazon_password, session_path,
args.headless)
if not items_csv or not orders_csv: # Refunds are optional
logger.critical('Order history either not provided at command line or '
'unable to fetch. Exiting.')
exit(1)
orders = amazon.Order.parse_from_csv(orders_csv,
ProgressCounter('Parsing Orders - '))
items = amazon.Item.parse_from_csv(items_csv,
ProgressCounter('Parsing Items - '))
refunds = ([] if not refunds_csv else amazon.Refund.parse_from_csv(
refunds_csv, ProgressCounter('Parsing Refunds - ')))
if args.dry_run:
logger.info('\nDry Run; no modifications being sent to Mint.\n')
# Initialize the stats. Explicitly initialize stats that might not be
# accumulated (conditionals).
stats = Counter(
adjust_itemized_tax=0,
already_up_to_date=0,
misc_charge=0,
new_tag=0,
no_retag=0,
retag=0,
user_skipped_retag=0,
personal_cat=0,
)
mint_client = MintClient(args.mint_email, args.mint_password, session_path,
args.headless, args.mint_mfa_method,
args.wait_for_sync)
if args.pickled_epoch:
mint_trans, mint_category_name_to_id = (
get_trans_and_categories_from_pickle(args.pickled_epoch,
args.mint_pickle_location))
else:
# Get the date of the oldest Amazon order.
if not start_date:
start_date = min([o.order_date for o in orders])
if refunds:
start_date = min(start_date,
min([o.order_date for o in refunds]))
# Double the length of transaction history to help aid in
# personalized category tagging overrides.
today = datetime.date.today()
start_date = today - (today - start_date) * 2
mint_category_name_to_id = mint_client.get_categories()
mint_transactions_json = mint_client.get_transactions(start_date)
epoch = int(time.time())
mint_trans = mint.Transaction.parse_from_json(mint_transactions_json)
dump_trans_and_categories(mint_trans, mint_category_name_to_id, epoch,
args.mint_pickle_location)
updates, unmatched_orders = tagger.get_mint_updates(
orders, items, refunds, mint_trans, args, stats,
mint_category_name_to_id)
log_amazon_stats(items, orders, refunds)
log_processing_stats(stats)
if args.print_unmatched and unmatched_orders:
logger.warning(
'The following were not matched to Mint transactions:\n')
by_oid = defaultdict(list)
for uo in unmatched_orders:
by_oid[uo.order_id].append(uo)
for unmatched_by_oid in by_oid.values():
orders = [o for o in unmatched_by_oid if o.is_debit]
refunds = [o for o in unmatched_by_oid if not o.is_debit]
if orders:
print_unmatched(amazon.Order.merge(orders))
for r in amazon.Refund.merge(refunds):
print_unmatched(r)
if not updates:
logger.info(
'All done; no new tags to be updated at this point in time!')
exit(0)
if args.dry_run:
logger.info('Dry run. Following are proposed changes:')
if args.skip_dry_print:
logger.info('Dry run print results skipped!')
else:
tagger.print_dry_run(updates,
ignore_category=args.no_tag_categories)
else:
mint_client.send_updates(updates,
ignore_category=args.no_tag_categories)
class TaggerWorker(QObject):
"""This class is required to prevent locking up the main Qt thread."""
on_error = pyqtSignal(str)
on_review_ready = pyqtSignal(tagger.UpdatesResult)
on_updates_sent = pyqtSignal(int)
on_stopped = pyqtSignal()
on_mint_mfa = pyqtSignal()
on_mint_mfa_done = pyqtSignal()
on_progress = pyqtSignal(str, int, int)
stopping = False
webdriver = None
@pyqtSlot()
def stop(self):
self.stopping = True
@pyqtSlot(str)
def mfa_code(self, code):
logger.info(code)
self.mfa_code = code
@pyqtSlot(object)
def create_updates(self, args, parent):
try:
self.do_create_updates(args, parent)
except Exception as e:
msg = 'Internal error while creating updates: {}'.format(e)
self.on_error.emit(msg)
logger.exception(msg)
@pyqtSlot(list, object)
def send_updates(self, updates, args):
try:
self.do_send_updates(updates, args)
except Exception as e:
msg = 'Internal error while sending updates: {}'.format(e)
self.on_error.emit(msg)
logger.exception(msg)
def close_webdriver(self):
if self.webdriver:
self.webdriver.close()
self.webdriver = None
def get_webdriver(self, args):
if self.webdriver:
logger.info('Using existing webdriver')
return self.webdriver
logger.info('Creating a new webdriver')
self.webdriver = get_webdriver(args.headless, args.session_path)
return self.webdriver
def do_create_updates(self, args, parent):
def on_mint_mfa(prompt):
logger.info('Asking for Mint MFA')
self.on_mint_mfa.emit()
loop = QEventLoop()
self.on_mint_mfa_done.connect(loop.quit)
loop.exec_()
logger.info(self.mfa_code)
return self.mfa_code
# Factory that handles indeterminite, determinite, and counter style.
def progress_factory(msg, max=0):
return QtProgress(msg, max, self.on_progress.emit)
atexit.register(self.close_webdriver)
bound_webdriver_factory = partial(self.get_webdriver, args)
self.mint_client = MintClient(args,
bound_webdriver_factory,
mfa_input_callback=on_mint_mfa)
if not fetch_order_history(args, bound_webdriver_factory,
progress_factory):
self.on_error.emit(
'Failed to fetch Amazon order history. Check credentials')
return
results = tagger.create_updates(
args,
self.mint_client,
on_critical=self.on_error.emit,
indeterminate_progress_factory=progress_factory,
determinate_progress_factory=progress_factory,
counter_progress_factory=progress_factory)
if results.success and not self.stopping:
self.on_review_ready.emit(results)
def do_send_updates(self, updates, args):
num_updates = self.mint_client.send_updates(
updates,
progress=QtProgress('Sending updates to Mint', len(updates),
self.on_progress.emit),
ignore_category=args.no_tag_categories)
self.close_webdriver()
self.on_updates_sent.emit(num_updates)
def main():
root_logger = logging.getLogger()
root_logger.addHandler(logging.StreamHandler())
# For helping remote debugging, also log to file.
# Developers should be vigilant to NOT log any PII, ever (including being
# mindful of what exceptions might be thrown).
log_directory = os.path.join(TAGGER_BASE_PATH, 'Tagger Logs')
os.makedirs(log_directory, exist_ok=True)
log_filename = os.path.join(
log_directory, '{}.log'.format(time.strftime("%Y-%m-%d_%H-%M-%S")))
root_logger.addHandler(logging.FileHandler(log_filename))
is_outdated, latest_version = check_outdated('mint-amazon-tagger', VERSION)
if is_outdated:
print('Please update your version by running:\n'
'pip3 install mint-amazon-tagger --upgrade\n\n')
parser = argparse.ArgumentParser(
description='Tag Mint transactions based on itemized Amazon history.')
define_cli_args(parser)
args = parser.parse_args()
if args.version:
print('mint-amazon-tagger {}\nBy: Jeff Prouty'.format(VERSION))
exit(0)
mint_client = MintClient(email=args.mint_email,
password=args.mint_password,
session_path=args.session_path,
headless=args.headless,
mfa_method=args.mint_mfa_method,
wait_for_sync=args.mint_wait_for_sync,
progress_factory=indeterminate_progress_cli)
if args.dry_run:
logger.info('\nDry Run; no modifications being sent to Mint.\n')
def on_critical(msg):
logger.critical(msg)
exit(1)
results = tagger.create_updates(
args,
mint_client,
on_critical=on_critical,
indeterminate_progress_factory=indeterminate_progress_cli,
determinate_progress_factory=determinate_progress_cli,
counter_progress_factory=counter_progress_cli)
if not results.success:
logger.critical('Uncaught error from create_updates. Exiting')
exit(1)
log_amazon_stats(results.items, results.orders, results.refunds)
log_processing_stats(results.stats)
if args.print_unmatched and results.unmatched_orders:
logger.warning(
'The following were not matched to Mint transactions:\n')
by_oid = defaultdict(list)
for uo in results.unmatched_orders:
by_oid[uo.order_id].append(uo)
for unmatched_by_oid in by_oid.values():
orders = [o for o in unmatched_by_oid if o.is_debit]
refunds = [o for o in unmatched_by_oid if not o.is_debit]
if orders:
print_unmatched(amazon.Order.merge(orders))
for r in amazon.Refund.merge(refunds):
print_unmatched(r)
if not results.updates:
logger.info(
'All done; no new tags to be updated at this point in time!')
exit(0)
if args.dry_run:
logger.info('Dry run. Following are proposed changes:')
if args.skip_dry_print:
logger.info('Dry run print results skipped!')
else:
tagger.print_dry_run(results.updates,
ignore_category=args.no_tag_categories)
else:
num_updates = mint_client.send_updates(
results.updates,
progress=determinate_progress_cli('Updating Mint',
max=len(results.updates)),
ignore_category=args.no_tag_categories)
logger.info('Sent {} updates to Mint'.format(num_updates))
def main():
root_logger = logging.getLogger()
root_logger.setLevel(logging.INFO)
root_logger.addHandler(logging.StreamHandler())
# Disable noisy log spam from filelock from within tldextract.
logging.getLogger("filelock").setLevel(logging.WARN)
# For helping remote debugging, also log to file.
# Developers should be vigilant to NOT log any PII, ever (including being
# mindful of what exceptions might be thrown).
log_directory = os.path.join(TAGGER_BASE_PATH, 'Tagger Logs')
os.makedirs(log_directory, exist_ok=True)
log_filename = os.path.join(log_directory, '{}.log'.format(
time.strftime("%Y-%m-%d_%H-%M-%S")))
root_logger.addHandler(logging.FileHandler(log_filename))
is_outdated, latest_version = check_outdated('mint-amazon-tagger', VERSION)
if is_outdated:
print('Please update your version by running:\n'
'pip3 install mint-amazon-tagger --upgrade\n\n')
parser = argparse.ArgumentParser(
description='Tag Mint transactions based on itemized Amazon history.')
define_cli_args(parser)
args = parser.parse_args()
if args.version:
print('mint-amazon-tagger {}\nBy: Jeff Prouty'.format(VERSION))
exit(0)
webdriver = None
def close_webdriver():
if webdriver:
webdriver.close()
atexit.register(close_webdriver)
def webdriver_factory():
nonlocal webdriver
if webdriver:
return webdriver
webdriver = get_webdriver(args.headless, args.session_path)
return webdriver
mint_client = MintClient(args, webdriver_factory)
# Attempt to fetch the order history if csv files are not already provided.
if not has_order_history_csv_files(args):
if not maybe_prompt_for_amazon_credentials(args):
logger.critical('Failed to get Amazon credentials.')
exit(1)
if not fetch_order_history(
args, webdriver_factory, indeterminate_progress_cli):
logger.critical('Failed to fetch Amazon order history.')
exit(1)
if args.dry_run:
logger.info('\nDry Run; no modifications being sent to Mint.\n')
def on_critical(msg):
logger.critical(msg)
exit(1)
maybe_prompt_for_mint_credentials(args)
results = tagger.create_updates(
args, mint_client,
on_critical=on_critical,
indeterminate_progress_factory=indeterminate_progress_cli,
determinate_progress_factory=determinate_progress_cli,
counter_progress_factory=counter_progress_cli)
if not results.success:
logger.critical('Uncaught error from create_updates. Exiting')
exit(1)
log_amazon_stats(results.items, results.orders, results.refunds)
log_processing_stats(results.stats)
if args.print_unmatched and results.unmatched_orders:
logger.warning(
'The following were not matched to Mint transactions:\n')
by_oid = defaultdict(list)
for uo in results.unmatched_orders:
by_oid[uo.order_id].append(uo)
for unmatched_by_oid in by_oid.values():
orders = [o for o in unmatched_by_oid if o.is_debit]
refunds = [o for o in unmatched_by_oid if not o.is_debit]
if orders:
print_unmatched(amazon.Order.merge(orders))
for r in amazon.Refund.merge(refunds):
print_unmatched(r)
if not results.updates:
logger.info(
'All done; no new tags to be updated at this point in time!')
exit(0)
if args.dry_run:
logger.info('Dry run. Following are proposed changes:')
if args.skip_dry_print:
logger.info('Dry run print results skipped!')
else:
tagger.print_dry_run(results.updates,
ignore_category=args.no_tag_categories)
else:
num_updates = mint_client.send_updates(
results.updates,
progress=determinate_progress_cli(
'Updating Mint',
max=len(results.updates)),
ignore_category=args.no_tag_categories)
logger.info('Sent {} updates to Mint'.format(num_updates))