def build_app(self):
log_config = self._get_log_config()
if log_config:
logging.config.dictConfig(log_config)
else:
logging.basicConfig(level=logging.DEBUG)
self._load_plugins()
apply_default_enabled(self.plugins)
apply_configs(self.plugins)
target_components = self._get_target_components()
publisher = self._get_publisher()
downloader = self._get_downloader()
timeout = self._get_extract_timeout()
tmp_dir = self._get_extract_tmp_dir()
engine = Engine(target_components,
self._get_format(),
timeout=timeout,
tmp_dir=tmp_dir)
consumer = self._get_consumer(publisher, downloader, engine)
for w in self._get_watchers():
if isinstance(w, EngineWatcher):
w.watch(engine)
if isinstance(w, ConsumerWatcher):
w.watch(consumer)
return consumer
def _handle_config(config):
if config:
with open(config) as f:
cfg = yaml.load(f, Loader=Loader)
load_packages(cfg.get("packages", []))
apply_default_enabled(cfg)
apply_configs(cfg)
def collect(manifest=default_manifest, tmp_path=None, compress=False):
"""
This is the collection entry point. It accepts a manifest, a temporary
directory in which to store output, and a boolean for optional compression.
Args:
manifest (str or dict): json document or dictionary containing the
collection manifest. See default_manifest for an example.
tmp_path (str): The temporary directory that will be used to create a
working directory for storing component output as well as the final
tar.gz if one is generated.
compress (boolean): True to create a tar.gz and remove the original
workspace containing output. False to leave the workspace without
creating a tar.gz
Returns:
The full path to the created tar.gz or workspace.
"""
manifest = load_manifest(manifest)
client = manifest.get("client", {})
plugins = manifest.get("plugins", {})
run_strategy = client.get("run_strategy", {"name": "parallel"})
load_packages(plugins.get("packages", []))
apply_default_enabled(plugins)
apply_configs(plugins)
apply_blacklist(client.get("blacklist", {}))
to_persist = get_to_persist(client.get("persist", set()))
hostname = call("hostname -f", env=SAFE_ENV).strip()
suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S")
relative_path = "insights-%s-%s" % (hostname, suffix)
tmp_path = tmp_path or tempfile.gettempdir()
output_path = os.path.join(tmp_path, relative_path)
fs.ensure_path(output_path)
fs.touch(os.path.join(output_path, "insights_archive.txt"))
broker = dr.Broker()
ctx = create_context(client.get("context", {}))
broker[ctx.__class__] = ctx
parallel = run_strategy.get("name") == "parallel"
pool_args = run_strategy.get("args", {})
with get_pool(parallel, pool_args) as pool:
h = Hydration(output_path, pool=pool)
broker.add_observer(h.make_persister(to_persist))
dr.run_all(broker=broker, pool=pool)
if compress:
return create_archive(output_path)
return output_path
def __init__(
self,
packages=None,
format=JsonFormat,
include_timings=False,
include_tracebacks=False,
target_components=None,
component_config=None,
):
load_default_plugins()
load_packages(packages or [])
config = component_config or {}
apply_default_enabled(config)
apply_configs(config)
target_components = [
dr.get_component(c) for c in target_components or []
]
self._Format = format
self._include_timings = include_timings
self._include_tracebacks = include_tracebacks
self._target_components = target_components or None
def build_app(self):
log_config = self._get_log_config()
if log_config:
logging.config.dictConfig(log_config)
else:
logging.basicConfig(level=logging.DEBUG)
self._load_plugins()
apply_default_enabled(self.plugins)
apply_configs(self.plugins)
publisher = self._get_publisher()
downloader = self._get_downloader()
engine = self._get_engine()
consumer = self._get_consumer(publisher, downloader, engine)
for w in self._get_watchers():
if isinstance(w, EngineWatcher):
w.watch(engine)
if isinstance(w, ConsumerWatcher):
w.watch(consumer)
return consumer
def configure(config):
if config:
with open(config) as f:
apply_configs(yaml.safe_load(f))
def collect(manifest=default_manifest, tmp_path=None, compress=False, rm_conf=None, client_timeout=None):
"""
This is the collection entry point. It accepts a manifest, a temporary
directory in which to store output, and a boolean for optional compression.
Args:
manifest (str or dict): json document or dictionary containing the
collection manifest. See default_manifest for an example.
tmp_path (str): The temporary directory that will be used to create a
working directory for storing component output as well as the final
tar.gz if one is generated.
compress (boolean): True to create a tar.gz and remove the original
workspace containing output. False to leave the workspace without
creating a tar.gz
rm_conf (dict): Client-provided python dict containing keys
"commands", "files", and "keywords", to be injected
into the manifest blacklist.
client_timeout (int): Client-provided command timeout value
Returns:
The full path to the created tar.gz or workspace.
"""
manifest = load_manifest(manifest)
client = manifest.get("client", {})
plugins = manifest.get("plugins", {})
run_strategy = client.get("run_strategy", {"name": "parallel"})
load_packages(plugins.get("packages", []))
apply_default_enabled(plugins)
apply_configs(plugins)
apply_blacklist(client.get("blacklist", {}))
# insights-client
if client_timeout:
try:
client['context']['args']['timeout'] = client_timeout
except LookupError:
log.warning('Could not set timeout option.')
rm_conf = rm_conf or {}
apply_blacklist(rm_conf)
for component in rm_conf.get('components', []):
if not dr.get_component_by_name(component):
log.warning('WARNING: Unknown component in blacklist: %s' % component)
else:
dr.set_enabled(component, enabled=False)
log.warning('WARNING: Skipping component: %s', component)
to_persist = get_to_persist(client.get("persist", set()))
try:
filters.load()
except IOError as e:
# could not load filters file
log.debug("No filters available: %s", str(e))
except AttributeError as e:
# problem parsing the filters
log.debug("Could not parse filters: %s", str(e))
try:
hostname = call("hostname -f", env=SAFE_ENV).strip()
except CalledProcessError:
# problem calling hostname -f
hostname = call("hostname", env=SAFE_ENV).strip()
suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S")
relative_path = "insights-%s-%s" % (hostname, suffix)
tmp_path = tmp_path or tempfile.gettempdir()
output_path = os.path.join(tmp_path, relative_path)
fs.ensure_path(output_path)
fs.touch(os.path.join(output_path, "insights_archive.txt"))
broker = dr.Broker()
ctx = create_context(client.get("context", {}))
broker[ctx.__class__] = ctx
parallel = run_strategy.get("name") == "parallel"
pool_args = run_strategy.get("args", {})
with get_pool(parallel, pool_args) as pool:
h = Hydration(output_path, pool=pool)
broker.add_observer(h.make_persister(to_persist))
dr.run_all(broker=broker, pool=pool)
if compress:
return create_archive(output_path)
return output_path
