def main(args=None, stdin=sys.stdin): if args is None: args = sys.argv[1:] options, args = parser.parse_args(args) if len(args) < 1: parser.error('You must give a DB_CONNECTION string') insert_count = int(options.batch) request_tracker = RequestTracker(args[0], options.table_prefix) done = False while not done: done = True for index, line in enumerate(stdin): try: request_tracker.import_apache_line( line, default_host=options.host, default_scheme=options.scheme) except ValueError, e: print >> sys.stdout, str(e) if not index % 1000: sys.stdout.write('.') sys.stdout.flush() if index > insert_count: done = False break sys.stdout.write('writing db...\n') sys.stdout.flush() def writer(i=None, total=0): if i and not i % 1000: sys.stdout.write('.') sys.stdout.flush() if i is None: sys.stdout.write('write...') sys.stdout.flush() request_tracker.write_pending(writer) if not done: sys.stdout.write('\ncontinuing...') sys.stdout.flush() import gc gc.collect()