def initialize_broker(path, context=None, broker=None): ctx = create_context(path, context=context) broker = broker or dr.Broker() if isinstance(ctx, ClusterArchiveContext): return ctx, broker broker[ctx.__class__] = ctx if isinstance(ctx, SerializedArchiveContext): h = Hydration(ctx.root) broker = h.hydrate(broker=broker) return ctx, broker
def collect(manifest=default_manifest, tmp_path=None, compress=False): """ This is the collection entry point. It accepts a manifest, a temporary directory in which to store output, and a boolean for optional compression. Args: manifest (str or dict): json document or dictionary containing the collection manifest. See default_manifest for an example. tmp_path (str): The temporary directory that will be used to create a working directory for storing component output as well as the final tar.gz if one is generated. compress (boolean): True to create a tar.gz and remove the original workspace containing output. False to leave the workspace without creating a tar.gz Returns: The full path to the created tar.gz or workspace. """ manifest = load_manifest(manifest) client = manifest.get("client", {}) plugins = manifest.get("plugins", {}) run_strategy = client.get("run_strategy", {"name": "parallel"}) load_packages(plugins.get("packages", [])) apply_default_enabled(plugins) apply_configs(plugins) apply_blacklist(client.get("blacklist", {})) to_persist = get_to_persist(client.get("persist", set())) hostname = call("hostname -f", env=SAFE_ENV).strip() suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S") relative_path = "insights-%s-%s" % (hostname, suffix) tmp_path = tmp_path or tempfile.gettempdir() output_path = os.path.join(tmp_path, relative_path) fs.ensure_path(output_path) fs.touch(os.path.join(output_path, "insights_archive.txt")) broker = dr.Broker() ctx = create_context(client.get("context", {})) broker[ctx.__class__] = ctx parallel = run_strategy.get("name") == "parallel" pool_args = run_strategy.get("args", {}) with get_pool(parallel, pool_args) as pool: h = Hydration(output_path, pool=pool) broker.add_observer(h.make_persister(to_persist)) dr.run_all(broker=broker, pool=pool) if compress: return create_archive(output_path) return output_path
def make_broker(ctx): broker = dr.Broker() broker[ctx.__class__] = ctx if isinstance(ctx, SerializedArchiveContext): h = Hydration(ctx.root) broker = h.hydrate(broker=broker) dr.run(datasources, broker=broker) del broker[ctx.__class__] return broker
def test_raw_file(): before = RawFileProvider(relative_path, root) broker = dr.Broker() broker[thing] = before tmp_path = mkdtemp() try: hydra = Hydration(tmp_path) hydra.dehydrate(thing, broker) after = hydra.hydrate()[thing] assert after.content == before.content finally: if tmp_path and os.path.exists(tmp_path): fs.remove(tmp_path)
def test_hydrate_one(): raw = { "name": dr.get_name(thing), "exec_time": 0.25, "ser_time": 0.05, "results": { "type": "insights.tests.test_serde.Foo", "object": { "a": 1, "b": 2 } } } h = Hydration() key, result, exec_time, ser_time = h._hydrate_one(raw) assert key is thing assert isinstance(result, Foo) assert exec_time == 0.25 assert ser_time == 0.05
def test_round_trip(): tmp_path = mkdtemp() try: h = Hydration(tmp_path) broker = dr.Broker() broker[thing] = Foo() broker.exec_times[thing] = 0.5 h.dehydrate(thing, broker) fn = ".".join([dr.get_name(thing), h.ser_name]) assert os.path.exists(os.path.join(h.meta_data, fn)) broker = h.hydrate() assert thing in broker assert broker.exec_times[thing] >= 0.5 foo = broker[thing] assert foo.a == 1 assert foo.b == 2 finally: pass if os.path.exists(tmp_path): fs.remove(tmp_path)
def test_hydrate_one_multiple_results(): raw = { "name": dr.get_name(thing), "exec_time": 0.5, "ser_time": 0.1, "results": [ { "type": "insights.tests.test_serde.Foo", "object": { "a": 1, "b": 2 } }, { "type": "insights.tests.test_serde.Foo", "object": { "a": 3, "b": 4 } }, ] } h = Hydration() key, result, exec_time, ser_time = h._hydrate_one(raw) assert key is thing assert len(result) == 2 assert exec_time == 0.5 assert ser_time == 0.1 assert result[0].a == 1 assert result[0].b == 2 assert result[1].a == 3 assert result[1].b == 4
def collect(manifest=default_manifest, tmp_path=None, compress=False, rm_conf=None, client_timeout=None): """ This is the collection entry point. It accepts a manifest, a temporary directory in which to store output, and a boolean for optional compression. Args: manifest (str or dict): json document or dictionary containing the collection manifest. See default_manifest for an example. tmp_path (str): The temporary directory that will be used to create a working directory for storing component output as well as the final tar.gz if one is generated. compress (boolean): True to create a tar.gz and remove the original workspace containing output. False to leave the workspace without creating a tar.gz rm_conf (dict): Client-provided python dict containing keys "commands", "files", and "keywords", to be injected into the manifest blacklist. client_timeout (int): Client-provided command timeout value Returns: The full path to the created tar.gz or workspace. """ manifest = load_manifest(manifest) client = manifest.get("client", {}) plugins = manifest.get("plugins", {}) run_strategy = client.get("run_strategy", {"name": "parallel"}) load_packages(plugins.get("packages", [])) apply_default_enabled(plugins) apply_configs(plugins) apply_blacklist(client.get("blacklist", {})) # insights-client if client_timeout: try: client['context']['args']['timeout'] = client_timeout except LookupError: log.warning('Could not set timeout option.') rm_conf = rm_conf or {} apply_blacklist(rm_conf) for component in rm_conf.get('components', []): if not dr.get_component_by_name(component): log.warning('WARNING: Unknown component in blacklist: %s' % component) else: dr.set_enabled(component, enabled=False) log.warning('WARNING: Skipping component: %s', component) to_persist = get_to_persist(client.get("persist", set())) try: filters.load() except IOError as e: # could not load filters file log.debug("No filters available: %s", str(e)) except AttributeError as e: # problem parsing the filters log.debug("Could not parse filters: %s", str(e)) try: hostname = call("hostname -f", env=SAFE_ENV).strip() except CalledProcessError: # problem calling hostname -f hostname = call("hostname", env=SAFE_ENV).strip() suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S") relative_path = "insights-%s-%s" % (hostname, suffix) tmp_path = tmp_path or tempfile.gettempdir() output_path = os.path.join(tmp_path, relative_path) fs.ensure_path(output_path) fs.touch(os.path.join(output_path, "insights_archive.txt")) broker = dr.Broker() ctx = create_context(client.get("context", {})) broker[ctx.__class__] = ctx parallel = run_strategy.get("name") == "parallel" pool_args = run_strategy.get("args", {}) with get_pool(parallel, pool_args) as pool: h = Hydration(output_path, pool=pool) broker.add_observer(h.make_persister(to_persist)) dr.run_all(broker=broker, pool=pool) if compress: return create_archive(output_path) return output_path