def load(self, fp): self.size = 0 file_size = os.fstat(fp.fileno()).st_size nblocks = 1 + (file_size - 1) // self.blocksize bar = IncrementalBar('Computing', max=nblocks) bar.suffix = '%(percent).1f%% - %(eta)ds' for block in bar.iter(file_read_iterator(fp, self.blocksize)): self.append(self._hash_block(block)) self.size += len(block)
if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( 'path', help='Path to directory containing measurement data' ) args = parser.parse_args() # Use given path as working directory for script so glob works from there os.chdir(args.path) total_lines = get_total_lines(glob.iglob('**/*.CSV', recursive=True)) total_size = int(total_lines / BUFFER_SIZE) bar = IncrementalBar( max=total_size, suffix='%(percent)d%% [ETA: %(eta_td)s]' ) bar.start() csv_glob = peekable(glob.iglob('**/*.CSV', recursive=True)) line_reader = buffered_line_reader( line_reader(csv_glob), buffer_size=BUFFER_SIZE ) with requests.Session() as session: # Iterate through the progress bar to auto update the bar for lines in bar.iter(line_reader): bar.message = csv_glob.peek(bar.message) response = session.post(URL, data=lines) assert response.status_code == 204, response.text
def get_mint_updates( orders, items, refunds, trans, args, stats, mint_category_name_to_id=category.DEFAULT_MINT_CATEGORIES_TO_IDS): def get_prefix(is_debit): return (args.description_prefix if is_debit else args.description_return_prefix) # Remove items from cancelled orders. items = [i for i in items if not i.is_cancelled()] # Remove items that haven't shipped yet (also aren't charged). items = [i for i in items if i.order_status == 'Shipped'] # Remove items with zero quantity (it happens!) items = [i for i in items if i.quantity > 0] # Make more Items such that every item is quantity 1. This is critical # prior to associate_items_with_orders such that items with non-1 # quantities split into different packages can be associated with the # appropriate order. items = [si for i in items for si in i.split_by_quantity()] itemProgress = IncrementalBar('Matching Amazon Items with Orders', max=len(items)) amazon.associate_items_with_orders(orders, items, itemProgress) itemProgress.finish() # Only match orders that have items. orders = [o for o in orders if o.items] trans = mint.Transaction.unsplit(trans) stats['trans'] = len(trans) # Skip t if the original description doesn't contain 'amazon' trans = [t for t in trans if 'amazon' in t.omerchant.lower()] stats['amazon_in_desc'] = len(trans) # Skip t if it's pending. trans = [t for t in trans if not t.is_pending] stats['pending'] = stats['amazon_in_desc'] - len(trans) # Skip t if a category filter is given and t does not match. if args.mint_input_categories_filter: whitelist = set(args.mint_input_categories_filter.lower().split(',')) trans = [t for t in trans if t.category.lower() in whitelist] # Match orders. orderMatchProgress = IncrementalBar('Matching Amazon Orders w/ Mint Trans', max=len(orders)) match_transactions(trans, orders, orderMatchProgress) orderMatchProgress.finish() unmatched_trans = [t for t in trans if not t.orders] # Match refunds. refundMatchProgress = IncrementalBar( 'Matching Amazon Refunds w/ Mint Trans', max=len(refunds)) match_transactions(unmatched_trans, refunds, refundMatchProgress) refundMatchProgress.finish() unmatched_orders = [o for o in orders if not o.matched] unmatched_trans = [t for t in trans if not t.orders] unmatched_refunds = [r for r in refunds if not r.matched] num_gift_card = len([ o for o in unmatched_orders if 'Gift Certificate' in o.payment_instrument_type ]) num_unshipped = len([o for o in unmatched_orders if not o.shipment_date]) matched_orders = [o for o in orders if o.matched] matched_trans = [t for t in trans if t.orders] matched_refunds = [r for r in refunds if r.matched] stats['trans_unmatch'] = len(unmatched_trans) stats['order_unmatch'] = len(unmatched_orders) stats['refund_unmatch'] = len(unmatched_refunds) stats['trans_match'] = len(matched_trans) stats['order_match'] = len(matched_orders) stats['refund_match'] = len(matched_refunds) stats['skipped_orders_gift_card'] = num_gift_card stats['skipped_orders_unshipped'] = num_unshipped merged_orders = [] merged_refunds = [] updateCounter = IncrementalBar('Determining Mint Updates') updates = [] for t in updateCounter.iter(matched_trans): if t.is_debit: order = amazon.Order.merge(t.orders) merged_orders.extend(orders) if order.attribute_subtotal_diff_to_misc_charge(): stats['misc_charge'] += 1 # It's nice when "free" shipping cancels out with the shipping # promo, even though there is tax on said free shipping. Spread # that out across the items instead. # if order.attribute_itemized_diff_to_shipping_tax(): # stats['add_shipping_tax'] += 1 if order.attribute_itemized_diff_to_per_item_tax(): stats['adjust_itemized_tax'] += 1 assert micro_usd_nearly_equal(t.amount, order.total_charged) assert micro_usd_nearly_equal(t.amount, order.total_by_subtotals()) assert micro_usd_nearly_equal(t.amount, order.total_by_items()) new_transactions = order.to_mint_transactions( t, skip_free_shipping=not args.verbose_itemize) else: refunds = amazon.Refund.merge(t.orders) merged_refunds.extend(refunds) new_transactions = [r.to_mint_transaction(t) for r in refunds] assert micro_usd_nearly_equal( t.amount, mint.Transaction.sum_amounts(new_transactions)) for nt in new_transactions: nt.update_category_id(mint_category_name_to_id) prefix = get_prefix(t.is_debit) summarize_single_item_order = (t.is_debit and len(order.items) == 1 and not args.verbose_itemize) if args.no_itemize or summarize_single_item_order: new_transactions = mint.summarize_new_trans( t, new_transactions, prefix) else: new_transactions = mint.itemize_new_trans(new_transactions, prefix) if mint.Transaction.old_and_new_are_identical( t, new_transactions, ignore_category=args.no_tag_categories): stats['already_up_to_date'] += 1 continue if t.merchant.startswith(prefix): if args.prompt_retag: if args.num_updates > 0 and len(updates) >= args.num_updates: break logger.info('\nTransaction already tagged:') print_dry_run([(t, new_transactions)], ignore_category=args.no_tag_categories) logger.info('\nUpdate tag to proposed? [Yn] ') action = readchar.readchar() if action == '': exit(1) if action not in ('Y', 'y', '\r', '\n'): stats['user_skipped_retag'] += 1 continue stats['retag'] += 1 elif not args.retag_changed: stats['no_retag'] += 1 continue else: stats['retag'] += 1 else: stats['new_tag'] += 1 updates.append((t, new_transactions)) if args.num_updates > 0: updates = updates[:args.num_updates] return updates
for bar_cls in (Bar, ChargingBar, FillingSquaresBar, FillingCirclesBar): suffix = '%(index)d/%(max)d [%(elapsed)d / %(eta)d / %(eta_td)s] (%(iter_value)s)' bar = bar_cls(bar_cls.__name__, suffix=suffix) for i in bar.iter(range(200, 400)): sleep() for bar_cls in (IncrementalBar, PixelBar, ShadyBar): suffix = '%(percent)d%% [%(elapsed_td)s / %(eta)d / %(eta_td)s]' with bar_cls(bar_cls.__name__, suffix=suffix, max=200) as bar: for i in range(200): bar.next() sleep() bar = IncrementalBar(bold('Corolored'), color='green') for i in bar.iter(range(200)): sleep() for spin in (Spinner, PieSpinner, MoonSpinner, LineSpinner, PixelSpinner): for i in spin(spin.__name__ + ' %(index)d ').iter(range(100)): sleep() for singleton in (Counter, Countdown, Stack, Pie): for i in singleton(singleton.__name__ + ' ').iter(range(100)): sleep() bar = IncrementalBar('Random', suffix='%(index)d') for i in range(100): bar.goto(random.randint(0, 100)) sleep() bar.finish()