def dehydrate(self, comp, broker): """ Saves a component in the given broker to the file system. """ if not self.meta_data: raise Exception("Hydration meta_path not set. Can't dehydrate.") if not self.created: fs.ensure_path(self.meta_data, mode=0o770) if self.data: fs.ensure_path(self.data, mode=0o770) self.created = True c = comp doc = None try: name = dr.get_name(c) value = broker.get(c) errors = [ t for e in broker.exceptions.get(c, []) for t in broker.tracebacks[e] ] doc = { "name": name, "exec_time": broker.exec_times.get(c), "errors": errors } try: start = time.time() doc["results"] = marshal(value, root=self.data) except Exception: errors.append(traceback.format_exc()) doc["results"] = None finally: doc["ser_time"] = time.time() - start except Exception as ex: log.exception(ex) else: if doc is not None and (doc["results"] or doc["errors"]): try: path = os.path.join(self.meta_data, name + "." + self.ser_name) with open(path, "w") as f: ser.dump(doc, f) except Exception as boom: log.error("Could not serialize %s to %s: %r" % (name, self.ser_name, boom)) if path: fs.remove(path)
def collect(manifest=default_manifest, tmp_path=None, compress=False): """ This is the collection entry point. It accepts a manifest, a temporary directory in which to store output, and a boolean for optional compression. Args: manifest (str or dict): json document or dictionary containing the collection manifest. See default_manifest for an example. tmp_path (str): The temporary directory that will be used to create a working directory for storing component output as well as the final tar.gz if one is generated. compress (boolean): True to create a tar.gz and remove the original workspace containing output. False to leave the workspace without creating a tar.gz Returns: The full path to the created tar.gz or workspace. """ manifest = load_manifest(manifest) client = manifest.get("client", {}) plugins = manifest.get("plugins", {}) apply_default_enabled(plugins.get("default_component_enabled", False)) load_packages(plugins.get("packages", [])) apply_blacklist(client.get("blacklist", {})) apply_configs(plugins) to_persist = get_to_persist(client.get("persist", set())) hostname = call("hostname -f", env=SAFE_ENV).strip() suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S") relative_path = "insights-%s-%s" % (hostname, suffix) tmp_path = tmp_path or tempfile.gettempdir() output_path = os.path.join(tmp_path, relative_path) fs.ensure_path(output_path) fs.touch(os.path.join(output_path, "insights_archive.txt")) broker = dr.Broker() ctx = create_context(client.get("context", {})) broker[ctx.__class__] = ctx h = Hydration(output_path) broker.add_observer(h.make_persister(to_persist)) list(dr.run_incremental(broker=broker)) if compress: return create_archive(output_path) return output_path
def collect(manifest=default_manifest, tmp_path=None, compress=False, rm_conf=None, client_timeout=None): """ This is the collection entry point. It accepts a manifest, a temporary directory in which to store output, and a boolean for optional compression. Args: manifest (str or dict): json document or dictionary containing the collection manifest. See default_manifest for an example. tmp_path (str): The temporary directory that will be used to create a working directory for storing component output as well as the final tar.gz if one is generated. compress (boolean): True to create a tar.gz and remove the original workspace containing output. False to leave the workspace without creating a tar.gz rm_conf (dict): Client-provided python dict containing keys "commands", "files", and "keywords", to be injected into the manifest blacklist. client_timeout (int): Client-provided command timeout value Returns: The full path to the created tar.gz or workspace. """ manifest = load_manifest(manifest) client = manifest.get("client", {}) plugins = manifest.get("plugins", {}) run_strategy = client.get("run_strategy", {"name": "parallel"}) load_packages(plugins.get("packages", [])) apply_default_enabled(plugins) apply_configs(plugins) apply_blacklist(client.get("blacklist", {})) # insights-client if client_timeout: try: client['context']['args']['timeout'] = client_timeout except LookupError: log.warning('Could not set timeout option.') rm_conf = rm_conf or {} apply_blacklist(rm_conf) for component in rm_conf.get('components', []): if not dr.get_component_by_name(component): log.warning('WARNING: Unknown component in blacklist: %s' % component) else: dr.set_enabled(component, enabled=False) log.warning('WARNING: Skipping component: %s', component) to_persist = get_to_persist(client.get("persist", set())) try: filters.load() except IOError as e: # could not load filters file log.debug("No filters available: %s", str(e)) except AttributeError as e: # problem parsing the filters log.debug("Could not parse filters: %s", str(e)) try: hostname = call("hostname -f", env=SAFE_ENV).strip() except CalledProcessError: # problem calling hostname -f hostname = call("hostname", env=SAFE_ENV).strip() suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S") relative_path = "insights-%s-%s" % (hostname, suffix) tmp_path = tmp_path or tempfile.gettempdir() output_path = os.path.join(tmp_path, relative_path) fs.ensure_path(output_path) fs.touch(os.path.join(output_path, "insights_archive.txt")) broker = dr.Broker() ctx = create_context(client.get("context", {})) broker[ctx.__class__] = ctx parallel = run_strategy.get("name") == "parallel" pool_args = run_strategy.get("args", {}) with get_pool(parallel, pool_args) as pool: h = Hydration(output_path, pool=pool) broker.add_observer(h.make_persister(to_persist)) dr.run_all(broker=broker, pool=pool) if compress: return create_archive(output_path) return output_path
def test_exists_ensure_path_exists(): path = os.path.expandvars('$PWD') assert os.path.exists(path) fs.ensure_path(path) assert os.path.exists(path)
def serialize_openshift_output(obj, root): rel = os.path.join("k8s", *obj.gvk) dst = os.path.join(root, rel) fs.ensure_path(os.path.dirname(dst)) obj.write(dst) return {"relative_path": rel}
def test_ensure_path_mode(mode, mode_dir): fs.ensure_path(mode_dir, int(mode, 8)) assert oct(os.stat(mode_dir).st_mode)[-3:] == mode
def write(self, dst): args = self.create_args() fs.ensure_path(os.path.dirname(dst)) if args: p = Pipeline(*args, timeout=self.timeout, env=self.create_env()) return p.write(dst, keep_rc=self.keep_rc)
def write(self, dst): fs.ensure_path(os.path.dirname(dst)) call([which("cp", env=SAFE_ENV), self.path, dst], env=SAFE_ENV)
def serialize_datasource_provider(obj, root): dst = os.path.join(root, obj.relative_path.lstrip("/")) fs.ensure_path(os.path.dirname(dst)) obj.write(dst) return {"relative_path": obj.relative_path}