def load(self, fp):
self.size = 0
file_size = os.fstat(fp.fileno()).st_size
nblocks = 1 + (file_size - 1) // self.blocksize
bar = IncrementalBar('Computing', max=nblocks)
bar.suffix = '%(percent).1f%% - %(eta)ds'
for block in bar.iter(file_read_iterator(fp, self.blocksize)):
self.append(self._hash_block(block))
self.size += len(block)
def load(self, fp):
self.size = 0
file_size = os.fstat(fp.fileno()).st_size
nblocks = 1 + (file_size - 1) // self.blocksize
bar = IncrementalBar('Computing', max=nblocks)
bar.suffix = '%(percent).1f%% - %(eta)ds'
for block in bar.iter(file_read_iterator(fp, self.blocksize)):
self.append(self._hash_block(block))
self.size += len(block)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'path', help='Path to directory containing measurement data'
)
args = parser.parse_args()
# Use given path as working directory for script so glob works from there
os.chdir(args.path)
total_lines = get_total_lines(glob.iglob('**/*.CSV', recursive=True))
total_size = int(total_lines / BUFFER_SIZE)
bar = IncrementalBar(
max=total_size, suffix='%(percent)d%% [ETA: %(eta_td)s]'
)
bar.start()
csv_glob = peekable(glob.iglob('**/*.CSV', recursive=True))
line_reader = buffered_line_reader(
line_reader(csv_glob), buffer_size=BUFFER_SIZE
)
with requests.Session() as session:
# Iterate through the progress bar to auto update the bar
for lines in bar.iter(line_reader):
bar.message = csv_glob.peek(bar.message)
response = session.post(URL, data=lines)
assert response.status_code == 204, response.text
def get_mint_updates(
orders,
items,
refunds,
trans,
args,
stats,
mint_category_name_to_id=category.DEFAULT_MINT_CATEGORIES_TO_IDS):
def get_prefix(is_debit):
return (args.description_prefix
if is_debit else args.description_return_prefix)
# Remove items from cancelled orders.
items = [i for i in items if not i.is_cancelled()]
# Remove items that haven't shipped yet (also aren't charged).
items = [i for i in items if i.order_status == 'Shipped']
# Remove items with zero quantity (it happens!)
items = [i for i in items if i.quantity > 0]
# Make more Items such that every item is quantity 1. This is critical
# prior to associate_items_with_orders such that items with non-1
# quantities split into different packages can be associated with the
# appropriate order.
items = [si for i in items for si in i.split_by_quantity()]
itemProgress = IncrementalBar('Matching Amazon Items with Orders',
max=len(items))
amazon.associate_items_with_orders(orders, items, itemProgress)
itemProgress.finish()
# Only match orders that have items.
orders = [o for o in orders if o.items]
trans = mint.Transaction.unsplit(trans)
stats['trans'] = len(trans)
# Skip t if the original description doesn't contain 'amazon'
trans = [t for t in trans if 'amazon' in t.omerchant.lower()]
stats['amazon_in_desc'] = len(trans)
# Skip t if it's pending.
trans = [t for t in trans if not t.is_pending]
stats['pending'] = stats['amazon_in_desc'] - len(trans)
# Skip t if a category filter is given and t does not match.
if args.mint_input_categories_filter:
whitelist = set(args.mint_input_categories_filter.lower().split(','))
trans = [t for t in trans if t.category.lower() in whitelist]
# Match orders.
orderMatchProgress = IncrementalBar('Matching Amazon Orders w/ Mint Trans',
max=len(orders))
match_transactions(trans, orders, orderMatchProgress)
orderMatchProgress.finish()
unmatched_trans = [t for t in trans if not t.orders]
# Match refunds.
refundMatchProgress = IncrementalBar(
'Matching Amazon Refunds w/ Mint Trans', max=len(refunds))
match_transactions(unmatched_trans, refunds, refundMatchProgress)
refundMatchProgress.finish()
unmatched_orders = [o for o in orders if not o.matched]
unmatched_trans = [t for t in trans if not t.orders]
unmatched_refunds = [r for r in refunds if not r.matched]
num_gift_card = len([
o for o in unmatched_orders
if 'Gift Certificate' in o.payment_instrument_type
])
num_unshipped = len([o for o in unmatched_orders if not o.shipment_date])
matched_orders = [o for o in orders if o.matched]
matched_trans = [t for t in trans if t.orders]
matched_refunds = [r for r in refunds if r.matched]
stats['trans_unmatch'] = len(unmatched_trans)
stats['order_unmatch'] = len(unmatched_orders)
stats['refund_unmatch'] = len(unmatched_refunds)
stats['trans_match'] = len(matched_trans)
stats['order_match'] = len(matched_orders)
stats['refund_match'] = len(matched_refunds)
stats['skipped_orders_gift_card'] = num_gift_card
stats['skipped_orders_unshipped'] = num_unshipped
merged_orders = []
merged_refunds = []
updateCounter = IncrementalBar('Determining Mint Updates')
updates = []
for t in updateCounter.iter(matched_trans):
if t.is_debit:
order = amazon.Order.merge(t.orders)
merged_orders.extend(orders)
if order.attribute_subtotal_diff_to_misc_charge():
stats['misc_charge'] += 1
# It's nice when "free" shipping cancels out with the shipping
# promo, even though there is tax on said free shipping. Spread
# that out across the items instead.
# if order.attribute_itemized_diff_to_shipping_tax():
# stats['add_shipping_tax'] += 1
if order.attribute_itemized_diff_to_per_item_tax():
stats['adjust_itemized_tax'] += 1
assert micro_usd_nearly_equal(t.amount, order.total_charged)
assert micro_usd_nearly_equal(t.amount, order.total_by_subtotals())
assert micro_usd_nearly_equal(t.amount, order.total_by_items())
new_transactions = order.to_mint_transactions(
t, skip_free_shipping=not args.verbose_itemize)
else:
refunds = amazon.Refund.merge(t.orders)
merged_refunds.extend(refunds)
new_transactions = [r.to_mint_transaction(t) for r in refunds]
assert micro_usd_nearly_equal(
t.amount, mint.Transaction.sum_amounts(new_transactions))
for nt in new_transactions:
nt.update_category_id(mint_category_name_to_id)
prefix = get_prefix(t.is_debit)
summarize_single_item_order = (t.is_debit and len(order.items) == 1
and not args.verbose_itemize)
if args.no_itemize or summarize_single_item_order:
new_transactions = mint.summarize_new_trans(
t, new_transactions, prefix)
else:
new_transactions = mint.itemize_new_trans(new_transactions, prefix)
if mint.Transaction.old_and_new_are_identical(
t, new_transactions, ignore_category=args.no_tag_categories):
stats['already_up_to_date'] += 1
continue
if t.merchant.startswith(prefix):
if args.prompt_retag:
if args.num_updates > 0 and len(updates) >= args.num_updates:
break
logger.info('\nTransaction already tagged:')
print_dry_run([(t, new_transactions)],
ignore_category=args.no_tag_categories)
logger.info('\nUpdate tag to proposed? [Yn] ')
action = readchar.readchar()
if action == '':
exit(1)
if action not in ('Y', 'y', '\r', '\n'):
stats['user_skipped_retag'] += 1
continue
stats['retag'] += 1
elif not args.retag_changed:
stats['no_retag'] += 1
continue
else:
stats['retag'] += 1
else:
stats['new_tag'] += 1
updates.append((t, new_transactions))
if args.num_updates > 0:
updates = updates[:args.num_updates]
return updates
for bar_cls in (Bar, ChargingBar, FillingSquaresBar, FillingCirclesBar):
suffix = '%(index)d/%(max)d [%(elapsed)d / %(eta)d / %(eta_td)s] (%(iter_value)s)'
bar = bar_cls(bar_cls.__name__, suffix=suffix)
for i in bar.iter(range(200, 400)):
sleep()
for bar_cls in (IncrementalBar, PixelBar, ShadyBar):
suffix = '%(percent)d%% [%(elapsed_td)s / %(eta)d / %(eta_td)s]'
with bar_cls(bar_cls.__name__, suffix=suffix, max=200) as bar:
for i in range(200):
bar.next()
sleep()
bar = IncrementalBar(bold('Corolored'), color='green')
for i in bar.iter(range(200)):
sleep()
for spin in (Spinner, PieSpinner, MoonSpinner, LineSpinner, PixelSpinner):
for i in spin(spin.__name__ + ' %(index)d ').iter(range(100)):
sleep()
for singleton in (Counter, Countdown, Stack, Pie):
for i in singleton(singleton.__name__ + ' ').iter(range(100)):
sleep()
bar = IncrementalBar('Random', suffix='%(index)d')
for i in range(100):
bar.goto(random.randint(0, 100))
sleep()
bar.finish()