def test_run(self):
def threadfunc(wfd):
wfd.write(u'hello 100\n')
wfd.flush()
wfd.write(u'hello 102\n')
wfd.flush()
time.sleep(0.11)
wfd.write(u'hello 103\n')
wfd.flush()
wfd.close()
r, w = os.pipe()
rfd = os.fdopen(r, u'r')
wfd = os.fdopen(w, u'w')
thd = threading.Thread(target=threadfunc, args=(wfd,))
thd.start()
sio = StringIO()
feed = delta.fd_feed(rfd, 0.1)
parser = delta.Parser(flex=True, absolute=False, use_colors=False)
printer = delta.Printer(sio, timestamps=False, separators=False, orig=False, skip_zeros=False)
delta.run(feed, parser, printer)
self.assertEqual(sio.getvalue(), u'''hello 100
hello +2
hello +1
''')
thd.join()
rfd.close()
def run(data_dir, scratch_dir, results_dir):
tick = time.time()
job = os.path.basename(scratch_dir)
print('Copying to scratch')
db_dir = copy_results(results_dir, scratch_dir)
warcs_dir = copy_data(data_dir, scratch_dir)
index_path = os.path.join(db_dir, 'index.db')
print('Indexing')
indexer.index(warcs_dir, index_path)
delta_dir = os.path.join(scratch_dir, 'warcs')
print('Creating delta versions')
deltas = delta.run(warcs_dir, delta_dir, db_dir, job)
deltas.append('no_delta')
print('Compressing')
compress.run(deltas, delta_dir, db_dir)
print('Copying results back to home')
save_results(db_dir, results_dir)
shutil.rmtree(scratch_dir)
with open(os.path.join(results_dir, 'time_taken.txt'), 'w') as fd:
fd.write('%d' % int(time.time() - tick))
with open(os.path.join(results_dir, 'DONE'), 'w') as fd:
fd.write('done\n')
