def build_app(self): log_config = self._get_log_config() if log_config: logging.config.dictConfig(log_config) else: logging.basicConfig(level=logging.DEBUG) self._load_plugins() apply_default_enabled(self.plugins) apply_configs(self.plugins) target_components = self._get_target_components() publisher = self._get_publisher() downloader = self._get_downloader() timeout = self._get_extract_timeout() tmp_dir = self._get_extract_tmp_dir() engine = Engine(target_components, self._get_format(), timeout=timeout, tmp_dir=tmp_dir) consumer = self._get_consumer(publisher, downloader, engine) for w in self._get_watchers(): if isinstance(w, EngineWatcher): w.watch(engine) if isinstance(w, ConsumerWatcher): w.watch(consumer) return consumer
def _handle_config(config): if config: with open(config) as f: cfg = yaml.load(f, Loader=Loader) load_packages(cfg.get("packages", [])) apply_default_enabled(cfg) apply_configs(cfg)
def collect(manifest=default_manifest, tmp_path=None, compress=False): """ This is the collection entry point. It accepts a manifest, a temporary directory in which to store output, and a boolean for optional compression. Args: manifest (str or dict): json document or dictionary containing the collection manifest. See default_manifest for an example. tmp_path (str): The temporary directory that will be used to create a working directory for storing component output as well as the final tar.gz if one is generated. compress (boolean): True to create a tar.gz and remove the original workspace containing output. False to leave the workspace without creating a tar.gz Returns: The full path to the created tar.gz or workspace. """ manifest = load_manifest(manifest) client = manifest.get("client", {}) plugins = manifest.get("plugins", {}) run_strategy = client.get("run_strategy", {"name": "parallel"}) load_packages(plugins.get("packages", [])) apply_default_enabled(plugins) apply_configs(plugins) apply_blacklist(client.get("blacklist", {})) to_persist = get_to_persist(client.get("persist", set())) hostname = call("hostname -f", env=SAFE_ENV).strip() suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S") relative_path = "insights-%s-%s" % (hostname, suffix) tmp_path = tmp_path or tempfile.gettempdir() output_path = os.path.join(tmp_path, relative_path) fs.ensure_path(output_path) fs.touch(os.path.join(output_path, "insights_archive.txt")) broker = dr.Broker() ctx = create_context(client.get("context", {})) broker[ctx.__class__] = ctx parallel = run_strategy.get("name") == "parallel" pool_args = run_strategy.get("args", {}) with get_pool(parallel, pool_args) as pool: h = Hydration(output_path, pool=pool) broker.add_observer(h.make_persister(to_persist)) dr.run_all(broker=broker, pool=pool) if compress: return create_archive(output_path) return output_path
def __init__( self, packages=None, format=JsonFormat, include_timings=False, include_tracebacks=False, target_components=None, component_config=None, ): load_default_plugins() load_packages(packages or []) config = component_config or {} apply_default_enabled(config) apply_configs(config) target_components = [ dr.get_component(c) for c in target_components or [] ] self._Format = format self._include_timings = include_timings self._include_tracebacks = include_tracebacks self._target_components = target_components or None
def build_app(self): log_config = self._get_log_config() if log_config: logging.config.dictConfig(log_config) else: logging.basicConfig(level=logging.DEBUG) self._load_plugins() apply_default_enabled(self.plugins) apply_configs(self.plugins) publisher = self._get_publisher() downloader = self._get_downloader() engine = self._get_engine() consumer = self._get_consumer(publisher, downloader, engine) for w in self._get_watchers(): if isinstance(w, EngineWatcher): w.watch(engine) if isinstance(w, ConsumerWatcher): w.watch(consumer) return consumer
def configure(config): if config: with open(config) as f: apply_configs(yaml.safe_load(f))
def collect(manifest=default_manifest, tmp_path=None, compress=False, rm_conf=None, client_timeout=None): """ This is the collection entry point. It accepts a manifest, a temporary directory in which to store output, and a boolean for optional compression. Args: manifest (str or dict): json document or dictionary containing the collection manifest. See default_manifest for an example. tmp_path (str): The temporary directory that will be used to create a working directory for storing component output as well as the final tar.gz if one is generated. compress (boolean): True to create a tar.gz and remove the original workspace containing output. False to leave the workspace without creating a tar.gz rm_conf (dict): Client-provided python dict containing keys "commands", "files", and "keywords", to be injected into the manifest blacklist. client_timeout (int): Client-provided command timeout value Returns: The full path to the created tar.gz or workspace. """ manifest = load_manifest(manifest) client = manifest.get("client", {}) plugins = manifest.get("plugins", {}) run_strategy = client.get("run_strategy", {"name": "parallel"}) load_packages(plugins.get("packages", [])) apply_default_enabled(plugins) apply_configs(plugins) apply_blacklist(client.get("blacklist", {})) # insights-client if client_timeout: try: client['context']['args']['timeout'] = client_timeout except LookupError: log.warning('Could not set timeout option.') rm_conf = rm_conf or {} apply_blacklist(rm_conf) for component in rm_conf.get('components', []): if not dr.get_component_by_name(component): log.warning('WARNING: Unknown component in blacklist: %s' % component) else: dr.set_enabled(component, enabled=False) log.warning('WARNING: Skipping component: %s', component) to_persist = get_to_persist(client.get("persist", set())) try: filters.load() except IOError as e: # could not load filters file log.debug("No filters available: %s", str(e)) except AttributeError as e: # problem parsing the filters log.debug("Could not parse filters: %s", str(e)) try: hostname = call("hostname -f", env=SAFE_ENV).strip() except CalledProcessError: # problem calling hostname -f hostname = call("hostname", env=SAFE_ENV).strip() suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S") relative_path = "insights-%s-%s" % (hostname, suffix) tmp_path = tmp_path or tempfile.gettempdir() output_path = os.path.join(tmp_path, relative_path) fs.ensure_path(output_path) fs.touch(os.path.join(output_path, "insights_archive.txt")) broker = dr.Broker() ctx = create_context(client.get("context", {})) broker[ctx.__class__] = ctx parallel = run_strategy.get("name") == "parallel" pool_args = run_strategy.get("args", {}) with get_pool(parallel, pool_args) as pool: h = Hydration(output_path, pool=pool) broker.add_observer(h.make_persister(to_persist)) dr.run_all(broker=broker, pool=pool) if compress: return create_archive(output_path) return output_path