def dehydrate(self, comp, broker):
"""
Saves a component in the given broker to the file system.
"""
if not self.meta_data:
raise Exception("Hydration meta_path not set. Can't dehydrate.")
if not self.created:
fs.ensure_path(self.meta_data, mode=0o770)
if self.data:
fs.ensure_path(self.data, mode=0o770)
self.created = True
c = comp
doc = None
try:
name = dr.get_name(c)
value = broker.get(c)
errors = [
t for e in broker.exceptions.get(c, [])
for t in broker.tracebacks[e]
]
doc = {
"name": name,
"exec_time": broker.exec_times.get(c),
"errors": errors
}
try:
start = time.time()
doc["results"] = marshal(value, root=self.data)
except Exception:
errors.append(traceback.format_exc())
doc["results"] = None
finally:
doc["ser_time"] = time.time() - start
except Exception as ex:
log.exception(ex)
else:
if doc is not None and (doc["results"] or doc["errors"]):
try:
path = os.path.join(self.meta_data,
name + "." + self.ser_name)
with open(path, "w") as f:
ser.dump(doc, f)
except Exception as boom:
log.error("Could not serialize %s to %s: %r" %
(name, self.ser_name, boom))
if path:
fs.remove(path)
def collect(manifest=default_manifest, tmp_path=None, compress=False):
"""
This is the collection entry point. It accepts a manifest, a temporary
directory in which to store output, and a boolean for optional compression.
Args:
manifest (str or dict): json document or dictionary containing the
collection manifest. See default_manifest for an example.
tmp_path (str): The temporary directory that will be used to create a
working directory for storing component output as well as the final
tar.gz if one is generated.
compress (boolean): True to create a tar.gz and remove the original
workspace containing output. False to leave the workspace without
creating a tar.gz
Returns:
The full path to the created tar.gz or workspace.
"""
manifest = load_manifest(manifest)
client = manifest.get("client", {})
plugins = manifest.get("plugins", {})
apply_default_enabled(plugins.get("default_component_enabled", False))
load_packages(plugins.get("packages", []))
apply_blacklist(client.get("blacklist", {}))
apply_configs(plugins)
to_persist = get_to_persist(client.get("persist", set()))
hostname = call("hostname -f", env=SAFE_ENV).strip()
suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S")
relative_path = "insights-%s-%s" % (hostname, suffix)
tmp_path = tmp_path or tempfile.gettempdir()
output_path = os.path.join(tmp_path, relative_path)
fs.ensure_path(output_path)
fs.touch(os.path.join(output_path, "insights_archive.txt"))
broker = dr.Broker()
ctx = create_context(client.get("context", {}))
broker[ctx.__class__] = ctx
h = Hydration(output_path)
broker.add_observer(h.make_persister(to_persist))
list(dr.run_incremental(broker=broker))
if compress:
return create_archive(output_path)
return output_path
def collect(manifest=default_manifest, tmp_path=None, compress=False, rm_conf=None, client_timeout=None):
"""
This is the collection entry point. It accepts a manifest, a temporary
directory in which to store output, and a boolean for optional compression.
Args:
manifest (str or dict): json document or dictionary containing the
collection manifest. See default_manifest for an example.
tmp_path (str): The temporary directory that will be used to create a
working directory for storing component output as well as the final
tar.gz if one is generated.
compress (boolean): True to create a tar.gz and remove the original
workspace containing output. False to leave the workspace without
creating a tar.gz
rm_conf (dict): Client-provided python dict containing keys
"commands", "files", and "keywords", to be injected
into the manifest blacklist.
client_timeout (int): Client-provided command timeout value
Returns:
The full path to the created tar.gz or workspace.
"""
manifest = load_manifest(manifest)
client = manifest.get("client", {})
plugins = manifest.get("plugins", {})
run_strategy = client.get("run_strategy", {"name": "parallel"})
load_packages(plugins.get("packages", []))
apply_default_enabled(plugins)
apply_configs(plugins)
apply_blacklist(client.get("blacklist", {}))
# insights-client
if client_timeout:
try:
client['context']['args']['timeout'] = client_timeout
except LookupError:
log.warning('Could not set timeout option.')
rm_conf = rm_conf or {}
apply_blacklist(rm_conf)
for component in rm_conf.get('components', []):
if not dr.get_component_by_name(component):
log.warning('WARNING: Unknown component in blacklist: %s' % component)
else:
dr.set_enabled(component, enabled=False)
log.warning('WARNING: Skipping component: %s', component)
to_persist = get_to_persist(client.get("persist", set()))
try:
filters.load()
except IOError as e:
# could not load filters file
log.debug("No filters available: %s", str(e))
except AttributeError as e:
# problem parsing the filters
log.debug("Could not parse filters: %s", str(e))
try:
hostname = call("hostname -f", env=SAFE_ENV).strip()
except CalledProcessError:
# problem calling hostname -f
hostname = call("hostname", env=SAFE_ENV).strip()
suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S")
relative_path = "insights-%s-%s" % (hostname, suffix)
tmp_path = tmp_path or tempfile.gettempdir()
output_path = os.path.join(tmp_path, relative_path)
fs.ensure_path(output_path)
fs.touch(os.path.join(output_path, "insights_archive.txt"))
broker = dr.Broker()
ctx = create_context(client.get("context", {}))
broker[ctx.__class__] = ctx
parallel = run_strategy.get("name") == "parallel"
pool_args = run_strategy.get("args", {})
with get_pool(parallel, pool_args) as pool:
h = Hydration(output_path, pool=pool)
broker.add_observer(h.make_persister(to_persist))
dr.run_all(broker=broker, pool=pool)
if compress:
return create_archive(output_path)
return output_path
def test_exists_ensure_path_exists():
path = os.path.expandvars('$PWD')
assert os.path.exists(path)
fs.ensure_path(path)
assert os.path.exists(path)
def serialize_openshift_output(obj, root):
rel = os.path.join("k8s", *obj.gvk)
dst = os.path.join(root, rel)
fs.ensure_path(os.path.dirname(dst))
obj.write(dst)
return {"relative_path": rel}
def test_ensure_path_mode(mode, mode_dir):
fs.ensure_path(mode_dir, int(mode, 8))
assert oct(os.stat(mode_dir).st_mode)[-3:] == mode
def write(self, dst):
args = self.create_args()
fs.ensure_path(os.path.dirname(dst))
if args:
p = Pipeline(*args, timeout=self.timeout, env=self.create_env())
return p.write(dst, keep_rc=self.keep_rc)
def write(self, dst):
fs.ensure_path(os.path.dirname(dst))
call([which("cp", env=SAFE_ENV), self.path, dst], env=SAFE_ENV)
def serialize_datasource_provider(obj, root):
dst = os.path.join(root, obj.relative_path.lstrip("/"))
fs.ensure_path(os.path.dirname(dst))
obj.write(dst)
return {"relative_path": obj.relative_path}
