def test_run(self): def threadfunc(wfd): wfd.write(u'hello 100\n') wfd.flush() wfd.write(u'hello 102\n') wfd.flush() time.sleep(0.11) wfd.write(u'hello 103\n') wfd.flush() wfd.close() r, w = os.pipe() rfd = os.fdopen(r, u'r') wfd = os.fdopen(w, u'w') thd = threading.Thread(target=threadfunc, args=(wfd,)) thd.start() sio = StringIO() feed = delta.fd_feed(rfd, 0.1) parser = delta.Parser(flex=True, absolute=False, use_colors=False) printer = delta.Printer(sio, timestamps=False, separators=False, orig=False, skip_zeros=False) delta.run(feed, parser, printer) self.assertEqual(sio.getvalue(), u'''hello 100 hello +2 hello +1 ''') thd.join() rfd.close()
def run(data_dir, scratch_dir, results_dir): tick = time.time() job = os.path.basename(scratch_dir) print('Copying to scratch') db_dir = copy_results(results_dir, scratch_dir) warcs_dir = copy_data(data_dir, scratch_dir) index_path = os.path.join(db_dir, 'index.db') print('Indexing') indexer.index(warcs_dir, index_path) delta_dir = os.path.join(scratch_dir, 'warcs') print('Creating delta versions') deltas = delta.run(warcs_dir, delta_dir, db_dir, job) deltas.append('no_delta') print('Compressing') compress.run(deltas, delta_dir, db_dir) print('Copying results back to home') save_results(db_dir, results_dir) shutil.rmtree(scratch_dir) with open(os.path.join(results_dir, 'time_taken.txt'), 'w') as fd: fd.write('%d' % int(time.time() - tick)) with open(os.path.join(results_dir, 'DONE'), 'w') as fd: fd.write('done\n')