def setUp(self):
self.dbname = 'test.db'
if os.path.isfile(self.dbname):
os.remove(self.dbname)
self.fixture = ProcessWebscope(self.dbname, log=False)
self.testfile = 'tests/data/minimal.gz'
def setUp(self):
if os.path.isfile('testbigger.db'):
os.remove('testbigger.db')
self.fixture = ProcessWebscope('testbigger.db', log=False)
"""
run_process.py
James Wang
Nov 26, 2014
"""
import os
import time
import datetime
from process_to_sqlite import ProcessWebscope
# process all files in the data directory
proc = ProcessWebscope('full.db', log=False)
skip = [4681992, 1936662, 0, 0, 0, 0, 0, 0, 0, 0]
counter = 0
t0 = time.time()
for file in os.listdir('Webscope/R6/'):
if file.endswith('.gz'):
print file
proc.process_file('Webscope/R6/' + file, skip_lines=skip[counter])
counter += 1
t1 = time.time()
the_time = str(datetime.timedelta(seconds=t1 - t0))
print('Processing all files took a total of {}'.format(the_time))
with open('gz_to_database_time.txt', 'w') as f:
f.write('Total time = {}'.format(the_time))
