class TestReferences: def __init__(self, docsdir, refsdir): self._docsdir = docsdir self._refsdir = refsdir self._skipped = get_skipped_tests(docsdir) self._test = Test() self.config = Config() self.printer = get_printer() try: os.makedirs(self._refsdir) except OSError as e: if e.errno != errno.EEXIST: raise except: raise def create_refs_for_file(self, filename, n_doc = 1, total_docs = 1): if filename in self._skipped: self.printer.print_default("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs)) return refs_path = os.path.join(self._refsdir, filename) try: os.makedirs(refs_path) except OSError as e: if e.errno != errno.EEXIST: raise except: raise doc_path = os.path.join(self._docsdir, filename) if not self.config.force and self._test.has_results(refs_path): self.printer.print_default("Results found, skipping '%s' (%d/%d)" % (doc_path, n_doc, total_docs)) return self.printer.printout_ln("Creating refs for '%s' (%d/%d)" % (doc_path, n_doc, total_docs)) if self._test.create_refs(doc_path, refs_path): self._test.create_checksums(refs_path, self.config.checksums_only) def create_refs(self): docs, total_docs = get_document_paths_from_dir(self._docsdir) n_doc = 0 for doc in docs: n_doc += 1 self.create_refs_for_file(doc, n_doc, total_docs)
class TestReferences: def __init__(self, docsdir, refsdir): self._docsdir = docsdir self._refsdir = refsdir self._skipped = get_skipped_tests(docsdir) self._test = Test() self.config = Config() self.printer = get_printer() self._total_tests = 1 self._n_tests = 0 self._queue = Queue() self._lock = RLock() try: os.makedirs(self._refsdir) except OSError as e: if e.errno != errno.EEXIST: raise except: raise def create_refs_for_file(self, filename): if filename in self._skipped: with self._lock: self._n_tests += 1 self.printer.print_default("Skipping test '%s'" % (os.path.join(self._docsdir, filename))) return refs_path = os.path.join(self._refsdir, filename) try: os.makedirs(refs_path) except OSError as e: if e.errno != errno.EEXIST: raise except: raise doc_path = os.path.join(self._docsdir, filename) if not self.config.force and self._test.has_results(refs_path): with self._lock: self._n_tests += 1 self.printer.print_default("Results found, skipping '%s'" % doc_path) return if self._test.create_refs(doc_path, refs_path): self._test.create_checksums(refs_path, self.config.checksums_only) with self._lock: self._n_tests += 1 self.printer.printout_ln("[%d/%d] %s: done" % (self._n_tests, self._total_tests, doc_path)) def _worker_thread(self): while True: doc = self._queue.get() self.create_refs_for_file(doc) self._queue.task_done() def create_refs(self): docs, total_docs = get_document_paths_from_dir(self._docsdir) self._total_tests = total_docs self.printer.printout_ln('Found %d documents' % (total_docs)) self.printer.printout_ln('Process %d using %d worker threads' % (os.getpid(), self.config.threads)) self.printer.printout_ln() self.printer.printout('Spawning %d workers...' % (self.config.threads)) for n_thread in range(self.config.threads): thread = Thread(target=self._worker_thread) thread.daemon = True thread.start() for doc in docs: self._queue.put(doc) self._queue.join()