def initialize_broker(path, context=None, broker=None):
ctx = create_context(path, context=context)
broker = broker or dr.Broker()
if isinstance(ctx, ClusterArchiveContext):
return ctx, broker
broker[ctx.__class__] = ctx
if isinstance(ctx, SerializedArchiveContext):
h = Hydration(ctx.root)
broker = h.hydrate(broker=broker)
return ctx, broker
def collect(manifest=default_manifest, tmp_path=None, compress=False):
"""
This is the collection entry point. It accepts a manifest, a temporary
directory in which to store output, and a boolean for optional compression.
Args:
manifest (str or dict): json document or dictionary containing the
collection manifest. See default_manifest for an example.
tmp_path (str): The temporary directory that will be used to create a
working directory for storing component output as well as the final
tar.gz if one is generated.
compress (boolean): True to create a tar.gz and remove the original
workspace containing output. False to leave the workspace without
creating a tar.gz
Returns:
The full path to the created tar.gz or workspace.
"""
manifest = load_manifest(manifest)
client = manifest.get("client", {})
plugins = manifest.get("plugins", {})
run_strategy = client.get("run_strategy", {"name": "parallel"})
load_packages(plugins.get("packages", []))
apply_default_enabled(plugins)
apply_configs(plugins)
apply_blacklist(client.get("blacklist", {}))
to_persist = get_to_persist(client.get("persist", set()))
hostname = call("hostname -f", env=SAFE_ENV).strip()
suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S")
relative_path = "insights-%s-%s" % (hostname, suffix)
tmp_path = tmp_path or tempfile.gettempdir()
output_path = os.path.join(tmp_path, relative_path)
fs.ensure_path(output_path)
fs.touch(os.path.join(output_path, "insights_archive.txt"))
broker = dr.Broker()
ctx = create_context(client.get("context", {}))
broker[ctx.__class__] = ctx
parallel = run_strategy.get("name") == "parallel"
pool_args = run_strategy.get("args", {})
with get_pool(parallel, pool_args) as pool:
h = Hydration(output_path, pool=pool)
broker.add_observer(h.make_persister(to_persist))
dr.run_all(broker=broker, pool=pool)
if compress:
return create_archive(output_path)
return output_path
def make_broker(ctx):
broker = dr.Broker()
broker[ctx.__class__] = ctx
if isinstance(ctx, SerializedArchiveContext):
h = Hydration(ctx.root)
broker = h.hydrate(broker=broker)
dr.run(datasources, broker=broker)
del broker[ctx.__class__]
return broker
def test_raw_file():
before = RawFileProvider(relative_path, root)
broker = dr.Broker()
broker[thing] = before
tmp_path = mkdtemp()
try:
hydra = Hydration(tmp_path)
hydra.dehydrate(thing, broker)
after = hydra.hydrate()[thing]
assert after.content == before.content
finally:
if tmp_path and os.path.exists(tmp_path):
fs.remove(tmp_path)
def test_hydrate_one():
raw = {
"name": dr.get_name(thing),
"exec_time": 0.25,
"ser_time": 0.05,
"results": {
"type": "insights.tests.test_serde.Foo",
"object": {
"a": 1,
"b": 2
}
}
}
h = Hydration()
key, result, exec_time, ser_time = h._hydrate_one(raw)
assert key is thing
assert isinstance(result, Foo)
assert exec_time == 0.25
assert ser_time == 0.05
def test_round_trip():
tmp_path = mkdtemp()
try:
h = Hydration(tmp_path)
broker = dr.Broker()
broker[thing] = Foo()
broker.exec_times[thing] = 0.5
h.dehydrate(thing, broker)
fn = ".".join([dr.get_name(thing), h.ser_name])
assert os.path.exists(os.path.join(h.meta_data, fn))
broker = h.hydrate()
assert thing in broker
assert broker.exec_times[thing] >= 0.5
foo = broker[thing]
assert foo.a == 1
assert foo.b == 2
finally:
pass
if os.path.exists(tmp_path):
fs.remove(tmp_path)
def test_hydrate_one_multiple_results():
raw = {
"name":
dr.get_name(thing),
"exec_time":
0.5,
"ser_time":
0.1,
"results": [
{
"type": "insights.tests.test_serde.Foo",
"object": {
"a": 1,
"b": 2
}
},
{
"type": "insights.tests.test_serde.Foo",
"object": {
"a": 3,
"b": 4
}
},
]
}
h = Hydration()
key, result, exec_time, ser_time = h._hydrate_one(raw)
assert key is thing
assert len(result) == 2
assert exec_time == 0.5
assert ser_time == 0.1
assert result[0].a == 1
assert result[0].b == 2
assert result[1].a == 3
assert result[1].b == 4
def collect(manifest=default_manifest, tmp_path=None, compress=False, rm_conf=None, client_timeout=None):
"""
This is the collection entry point. It accepts a manifest, a temporary
directory in which to store output, and a boolean for optional compression.
Args:
manifest (str or dict): json document or dictionary containing the
collection manifest. See default_manifest for an example.
tmp_path (str): The temporary directory that will be used to create a
working directory for storing component output as well as the final
tar.gz if one is generated.
compress (boolean): True to create a tar.gz and remove the original
workspace containing output. False to leave the workspace without
creating a tar.gz
rm_conf (dict): Client-provided python dict containing keys
"commands", "files", and "keywords", to be injected
into the manifest blacklist.
client_timeout (int): Client-provided command timeout value
Returns:
The full path to the created tar.gz or workspace.
"""
manifest = load_manifest(manifest)
client = manifest.get("client", {})
plugins = manifest.get("plugins", {})
run_strategy = client.get("run_strategy", {"name": "parallel"})
load_packages(plugins.get("packages", []))
apply_default_enabled(plugins)
apply_configs(plugins)
apply_blacklist(client.get("blacklist", {}))
# insights-client
if client_timeout:
try:
client['context']['args']['timeout'] = client_timeout
except LookupError:
log.warning('Could not set timeout option.')
rm_conf = rm_conf or {}
apply_blacklist(rm_conf)
for component in rm_conf.get('components', []):
if not dr.get_component_by_name(component):
log.warning('WARNING: Unknown component in blacklist: %s' % component)
else:
dr.set_enabled(component, enabled=False)
log.warning('WARNING: Skipping component: %s', component)
to_persist = get_to_persist(client.get("persist", set()))
try:
filters.load()
except IOError as e:
# could not load filters file
log.debug("No filters available: %s", str(e))
except AttributeError as e:
# problem parsing the filters
log.debug("Could not parse filters: %s", str(e))
try:
hostname = call("hostname -f", env=SAFE_ENV).strip()
except CalledProcessError:
# problem calling hostname -f
hostname = call("hostname", env=SAFE_ENV).strip()
suffix = datetime.utcnow().strftime("%Y%m%d%H%M%S")
relative_path = "insights-%s-%s" % (hostname, suffix)
tmp_path = tmp_path or tempfile.gettempdir()
output_path = os.path.join(tmp_path, relative_path)
fs.ensure_path(output_path)
fs.touch(os.path.join(output_path, "insights_archive.txt"))
broker = dr.Broker()
ctx = create_context(client.get("context", {}))
broker[ctx.__class__] = ctx
parallel = run_strategy.get("name") == "parallel"
pool_args = run_strategy.get("args", {})
with get_pool(parallel, pool_args) as pool:
h = Hydration(output_path, pool=pool)
broker.add_observer(h.make_persister(to_persist))
dr.run_all(broker=broker, pool=pool)
if compress:
return create_archive(output_path)
return output_path