def execute(self): thrift_targets = self.get_targets(self._is_thrift) with self.invalidated(thrift_targets) as invalidation_check: if not invalidation_check.invalid_vts: return with self.context.new_workunit( 'parallel-thrift-linter') as workunit: worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self.get_options().worker_count) scrooge_linter_classpath = self.tool_classpath( 'scrooge-linter') results = [] errors = [] for vt in invalidation_check.invalid_vts: r = worker_pool.submit_async_work( Work(self._lint, [(vt.target, scrooge_linter_classpath)])) results.append((r, vt)) for r, vt in results: r.wait() # MapResult will raise _value in `get` if the run is not successful. try: r.get() except ThriftLintError as e: errors.append(str(e)) else: vt.update() if errors: raise TaskError('\n'.join(errors))
def execute(self): if self.get_options().skip: return thrift_targets = self.context.targets(self._is_thrift) with self.invalidated(thrift_targets) as invalidation_check: if not invalidation_check.invalid_vts: return with self.context.new_workunit('parallel-thrift-linter') as workunit: worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self.get_options().worker_count) scrooge_linter_classpath = self.tool_classpath('scrooge-linter') results = [] errors = [] for vt in invalidation_check.invalid_vts: r = worker_pool.submit_async_work(Work(self._lint, [(vt.target, scrooge_linter_classpath)])) results.append((r, vt)) for r, vt in results: r.wait() # MapResult will raise _value in `get` if the run is not successful. try: r.get() except ThriftLintError as e: errors.append(str(e)) else: vt.update() if errors: raise TaskError('\n'.join(errors))
def background_worker_pool(self): if self._background_worker_pool is None: # Initialize lazily. self._background_worker_pool = WorkerPool( parent_workunit=self.get_background_root_workunit(), run_tracker=self, num_workers=self._num_background_workers) return self._background_worker_pool
def prepare_compile(self, cache_manager, all_targets, relevant_targets): super(JvmCompileIsolatedStrategy, self).prepare_compile(cache_manager, all_targets, relevant_targets) # Update the classpath by adding relevant target's classes directories to its classpath. compile_classpaths = self.context.products.get_data( 'compile_classpath') with self.context.new_workunit('validate-{}-analysis'.format( self._compile_task_name)): for target in relevant_targets: cc = self.compile_context(target) safe_mkdir(cc.classes_dir) compile_classpaths.add_for_target(target, [(conf, cc.classes_dir) for conf in self._confs]) self.validate_analysis(cc.analysis_file) # This ensures the workunit for the worker pool is set with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._compile_task_name)) \ as workunit: # This uses workunit.parent as the WorkerPool's parent so that child workunits # of different pools will show up in order in the html output. This way the current running # workunit is on the bottom of the page rather than possibly in the middle. self._worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self._worker_count)
def do_compile(self, invalidation_check, compile_contexts): """Executes compilations for the invalid targets contained in a single chunk.""" invalid_targets = [vt.target for vt in invalidation_check.invalid_vts] assert invalid_targets, "compile_chunk should only be invoked if there are invalid targets." # This ensures the workunit for the worker pool is set before attempting to compile. with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self.name())) \ as workunit: # This uses workunit.parent as the WorkerPool's parent so that child workunits # of different pools will show up in order in the html output. This way the current running # workunit is on the bottom of the page rather than possibly in the middle. worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self._worker_count) # Prepare the output directory for each invalid target, and confirm that analysis is valid. for target in invalid_targets: cc = self.select_runtime_context(compile_contexts[target]) safe_mkdir(cc.classes_dir) # Now create compile jobs for each invalid target one by one, using the classpath # generated by upstream JVM tasks and our own prepare_compile(). jobs = self._create_compile_jobs(compile_contexts, invalid_targets, invalidation_check.invalid_vts) exec_graph = ExecutionGraph(jobs, self.get_options().print_exception_stacktrace) try: exec_graph.execute(worker_pool, self.context.log) except ExecutionFailure as e: raise TaskError("Compilation failure: {}".format(e))
def prepare_execute(self, chunks): relevant_targets = list(itertools.chain(*chunks)) # Target -> sources (relative to buildroot). # TODO(benjy): Should sources_by_target be available in all Tasks? self._sources_by_target = self._compute_sources_by_target( relevant_targets) # Update the classpath by adding relevant target's classes directories to its classpath. compile_classpath = self.context.products.get_data('compile_classpath') runtime_classpath = self.context.products.get_data( 'runtime_classpath', compile_classpath.copy) with self.context.new_workunit('validate-{}-analysis'.format( self._name)): for target in relevant_targets: cc = self.compile_context(target) safe_mkdir(cc.classes_dir) runtime_classpath.add_for_target(target, [(conf, cc.classes_dir) for conf in self._confs]) self.validate_analysis(cc.analysis_file) # This ensures the workunit for the worker pool is set with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._name)) \ as workunit: # This uses workunit.parent as the WorkerPool's parent so that child workunits # of different pools will show up in order in the html output. This way the current running # workunit is on the bottom of the page rather than possibly in the middle. self._worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self._worker_count)
def foreground_worker_pool(self): if self._foreground_worker_pool is None: # Initialize lazily. self._foreground_worker_pool = WorkerPool( parent_workunit=self._main_root_workunit, run_tracker=self, num_workers=self._num_foreground_workers) return self._foreground_worker_pool
def execute(self): thrift_targets = self.get_targets(self._is_thrift) task_worker_count_configured = not self.get_options().is_default( "worker_count") subsystem_worker_count_configured = not ScroogeLinter.global_instance( ).options.is_default("worker_count") if task_worker_count_configured and subsystem_worker_count_configured: self.raise_conflicting_option("worker_count") worker_count = (self.get_options().worker_count if task_worker_count_configured else ScroogeLinter.global_instance().options.worker_count) with self.invalidated(thrift_targets) as invalidation_check: if not invalidation_check.invalid_vts: return with self.context.new_workunit( 'parallel-thrift-linter') as workunit: worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, worker_count, workunit.name) scrooge_linter_classpath = self.tool_classpath( 'scrooge-linter') results = [] errors = [] for vt in invalidation_check.invalid_vts: r = worker_pool.submit_async_work( Work(self._lint, [(vt.target, scrooge_linter_classpath)])) results.append((r, vt)) for r, vt in results: r.wait() # MapResult will raise _value in `get` if the run is not successful. try: r.get() except ThriftLintError as e: errors.append(str(e)) else: vt.update() if errors: raise TaskError('\n'.join(errors))
def do_compile(self, invalidation_check, compile_contexts, classpath_product): """Executes compilations for the invalid targets contained in a single chunk.""" invalid_targets = [vt.target for vt in invalidation_check.invalid_vts] valid_targets = [ vt.target for vt in invalidation_check.all_vts if vt.valid ] if self.execution_strategy == self.HERMETIC: self._set_directory_digests_for_valid_target_classpath_directories( valid_targets, compile_contexts) for valid_target in valid_targets: cc = self.select_runtime_context(compile_contexts[valid_target]) classpath_product.add_for_target( valid_target, [(conf, self._classpath_for_context(cc)) for conf in self._confs], ) self.register_extra_products_from_contexts(valid_targets, compile_contexts) if not invalid_targets: return # This ensures the workunit for the worker pool is set before attempting to compile. with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self.name())) \ as workunit: # This uses workunit.parent as the WorkerPool's parent so that child workunits # of different pools will show up in order in the html output. This way the current running # workunit is on the bottom of the page rather than possibly in the middle. worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self._worker_count) # Prepare the output directory for each invalid target, and confirm that analysis is valid. for target in invalid_targets: cc = self.select_runtime_context(compile_contexts[target]) safe_mkdir(cc.classes_dir.path) # Now create compile jobs for each invalid target one by one, using the classpath # generated by upstream JVM tasks and our own prepare_compile(). jobs = self._create_compile_jobs(compile_contexts, invalid_targets, invalidation_check.invalid_vts, classpath_product) exec_graph = ExecutionGraph( jobs, self.get_options().print_exception_stacktrace) try: exec_graph.execute(worker_pool, self.context.log) except ExecutionFailure as e: raise TaskError("Compilation failure: {}".format(e))
def test_keyboard_interrupts_propagated(self): condition = threading.Condition() condition.acquire() with self.assertRaises(KeyboardInterrupt): with temporary_dir() as rundir: pool = WorkerPool(WorkUnit(rundir, None, "work"), FakeRunTracker(), 1) try: pool.submit_async_work(Work(keyboard_interrupt_raiser, [()])) condition.wait(2) finally: pool.abort()
def prepare_execute(self, chunks): relevant_targets = list(itertools.chain(*chunks)) # Target -> sources (relative to buildroot). # TODO(benjy): Should sources_by_target be available in all Tasks? self._sources_by_target = self._compute_sources_by_target( relevant_targets) # Clone the compile_classpath to the runtime_classpath. compile_classpath = self.context.products.get_data('compile_classpath') runtime_classpath = self.context.products.get_data( 'runtime_classpath', compile_classpath.copy) # This ensures the workunit for the worker pool is set with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._name)) \ as workunit: # This uses workunit.parent as the WorkerPool's parent so that child workunits # of different pools will show up in order in the html output. This way the current running # workunit is on the bottom of the page rather than possibly in the middle. self._worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self._worker_count)
def execute(self): # In case we have no relevant targets and return early create the requested product maps. self._create_empty_products() def select_target(target): return self.select(target) relevant_targets = list(self.context.targets(predicate=select_target)) if not relevant_targets: return # Clone the compile_classpath to the runtime_classpath. compile_classpath = self.context.products.get_data('compile_classpath') runtime_classpath = self.context.products.get_data( 'runtime_classpath', compile_classpath.copy) # This ensures the workunit for the worker pool is set with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._name)) \ as workunit: # This uses workunit.parent as the WorkerPool's parent so that child workunits # of different pools will show up in order in the html output. This way the current running # workunit is on the bottom of the page rather than possibly in the middle. self._worker_pool = WorkerPool(workunit.parent, self.context.run_tracker, self._worker_count) classpath_product = self.context.products.get_data('runtime_classpath') fingerprint_strategy = self._fingerprint_strategy(classpath_product) # Invalidation check. Everything inside the with block must succeed for the # invalid targets to become valid. with self.invalidated(relevant_targets, invalidate_dependents=True, partition_size_hint=0, fingerprint_strategy=fingerprint_strategy, topological_order=True) as invalidation_check: # Initialize the classpath for all targets. compile_contexts = { vt.target: self._compile_context(vt.target, vt.results_dir) for vt in invalidation_check.all_vts } for cc in compile_contexts.values(): classpath_product.add_for_target(cc.target, [(conf, cc.classes_dir) for conf in self._confs]) # Register products for valid targets. valid_targets = [ vt.target for vt in invalidation_check.all_vts if vt.valid ] self._register_vts([compile_contexts[t] for t in valid_targets]) # Build any invalid targets (which will register products in the background). if invalidation_check.invalid_vts: invalid_targets = [ vt.target for vt in invalidation_check.invalid_vts ] self.compile_chunk( invalidation_check, compile_contexts, invalid_targets, self.extra_compile_time_classpath_elements()) # Once compilation has completed, replace the classpath entry for each target with # its jar'd representation. classpath_products = self.context.products.get_data( 'runtime_classpath') for cc in compile_contexts.values(): for conf in self._confs: classpath_products.remove_for_target( cc.target, [(conf, cc.classes_dir)]) classpath_products.add_for_target(cc.target, [(conf, cc.jar_file)])