def main(args=None, stdin=sys.stdin):
if args is None:
args = sys.argv[1:]
options, args = parser.parse_args(args)
if len(args) < 1:
parser.error('You must give a DB_CONNECTION string')
insert_count = int(options.batch)
request_tracker = RequestTracker(args[0], options.table_prefix)
done = False
while not done:
done = True
for index, line in enumerate(stdin):
try:
request_tracker.import_apache_line(
line,
default_host=options.host,
default_scheme=options.scheme)
except ValueError, e:
print >> sys.stdout, str(e)
if not index % 1000:
sys.stdout.write('.')
sys.stdout.flush()
if index > insert_count:
done = False
break
sys.stdout.write('writing db...\n')
sys.stdout.flush()
def writer(i=None, total=0):
if i and not i % 1000:
sys.stdout.write('.')
sys.stdout.flush()
if i is None:
sys.stdout.write('write...')
sys.stdout.flush()
request_tracker.write_pending(writer)
if not done:
sys.stdout.write('\ncontinuing...')
sys.stdout.flush()
import gc
gc.collect()
