def test_fetch_dependencies_unpack_parallel(self): output_path = tempfile.mkdtemp() save_dir = tempfile.mkdtemp() # use default parallelism of 4 for test pool = multiprocessing.Pool(4) target_objs = [{ "dependencies": [ { "type": "https", "source": "https://github.com/BurdenBear/kube-charts-mirror/raw/master/docs/nfs-client-provisioner-1.2.8.tgz", "output_path": "nfs-client-provisioner", "unpack": True, }, { "type": "https", "source": "https://github.com/BurdenBear/kube-charts-mirror/raw/master/docs/prometheus-pushgateway-1.2.13.tgz", "output_path": "prometheus-pushgateway", "unpack": True, }, ] }] try: fetch_dependencies(output_path, target_objs, save_dir, False, pool) pool.close() except Exception as e: pool.terminate() raise e finally: pool.join() for obj in target_objs[0]["dependencies"]: self.assertTrue( os.path.isdir(os.path.join(output_path, obj["output_path"]))) self.assertTrue( os.path.isdir(os.path.join(save_dir, obj["output_path"]))) rmtree(output_path) rmtree(save_dir)
def compile_targets(inventory_path, search_paths, output_path, parallel, targets, labels, ref_controller, **kwargs): """ Searches and loads target files, and runs compile_target() on a multiprocessing pool with parallel number of processes. kwargs are passed to compile_target() """ # temp_path will hold compiled items temp_path = tempfile.mkdtemp(suffix=".kapitan") updated_targets = targets try: updated_targets = search_targets(inventory_path, targets, labels) except CompileError as e: logger.error(e) sys.exit(1) # If --cache is set if kwargs.get("cache"): additional_cache_paths = kwargs.get("cache_paths") generate_inv_cache_hashes(inventory_path, targets, additional_cache_paths) if not targets: updated_targets = changed_targets(inventory_path, output_path) logger.debug("Changed targets since last compilation: %s", updated_targets) if len(updated_targets) == 0: logger.info("No changes since last compilation.") return pool = multiprocessing.Pool(parallel) try: target_objs = load_target_inventory(inventory_path, updated_targets) # append "compiled" to output_path so we can safely overwrite it compile_path = os.path.join(output_path, "compiled") worker = partial( compile_target, search_paths=search_paths, compile_path=temp_path, ref_controller=ref_controller, **kwargs, ) if not target_objs: raise CompileError("Error: no targets found") if kwargs.get("fetch_dependencies", False): fetch_dependencies(target_objs, pool) # compile_target() returns None on success # so p is only not None when raising an exception [p.get() for p in pool.imap_unordered(worker, target_objs) if p] os.makedirs(compile_path, exist_ok=True) # if '-t' is set on compile or only a few changed, only override selected targets if updated_targets: for target in updated_targets: compile_path_target = os.path.join(compile_path, target) temp_path_target = os.path.join(temp_path, target) os.makedirs(compile_path_target, exist_ok=True) shutil.rmtree(compile_path_target) shutil.copytree(temp_path_target, compile_path_target) logger.debug("Copied %s into %s", temp_path_target, compile_path_target) # otherwise override all targets else: shutil.rmtree(compile_path) shutil.copytree(temp_path, compile_path) logger.debug("Copied %s into %s", temp_path, compile_path) # validate the compiled outputs if kwargs.get("validate", False): validate_map = create_validate_mapping(target_objs, compile_path) worker = partial(schema_validate_kubernetes_output, cache_dir=kwargs.get("schemas_path", "./schemas")) [ p.get() for p in pool.imap_unordered(worker, validate_map.items()) if p ] # Save inventory and folders cache save_inv_cache(compile_path, targets) pool.close() except ReclassException as e: if isinstance(e, NotFoundError): logger.error("Inventory reclass error: inventory not found") else: logger.error("Inventory reclass error: %s", e.message) raise InventoryError(e.message) except Exception as e: # if compile worker fails, terminate immediately pool.terminate() logger.debug("Compile pool terminated") # only print traceback for errors we don't know about if not isinstance(e, KapitanError): logger.exception("Unknown (Non-Kapitan) Error occurred") logger.error("\n") logger.error(e) sys.exit(1) finally: # always wait for other worker processes to terminate pool.join() shutil.rmtree(temp_path) logger.debug("Removed %s", temp_path)