def _get_system_facts(archive_path):
facts = {}
default_packages = (
"insights.specs.default",
"insights.specs.insights_archive",
"insights.combiners",
"insights.parsers"
)
for pkg in default_packages:
dr.load_components(pkg)
broker = dr.Broker()
try:
with archives.extract(archive_path) as ex:
ctx = create_context(ex.tmp_dir, HostArchiveContext)
broker[ctx.__class__] = ctx
broker = dr.run(components=[Specs.machine_id, LsEtc],
broker=broker)
if Specs.machine_id in broker:
facts["id"] = broker[Specs.machine_id].content[0].strip()
if LsEtc in broker:
facts["satellite_managed"] = any([broker[LsEtc].dir_contains(*satellite_file)
for satellite_file in SATELLITE_MANAGED_FILES.values()
if satellite_file[0] in broker[LsEtc]])
except InvalidContentType:
LOGGER.error("Unable to parse archive.")
return facts
def process(self, broker, path):
for w in self.watchers:
w.watch_broker(broker)
result = None
try:
self.fire("pre_extract", broker, path)
with extract(path,
timeout=self.extract_timeout,
extract_dir=self.extract_tmp_dir) as extraction:
ctx, broker = initialize_broker(extraction.tmp_dir,
broker=broker)
self.fire("on_extract", ctx, broker, extraction)
output = StringIO()
with self.Formatter(broker, stream=output):
dr.run(self.target_components, broker=broker)
output.seek(0)
result = output.read()
self.fire("on_engine_success", broker, result)
return result
except Exception as ex:
self.fire("on_engine_failure", broker, ex)
raise
finally:
self.fire("on_engine_complete", broker)
def process(self, broker, path):
for w in self.watchers:
w.watch_broker(broker)
result = None
try:
self.fire("pre_extract", broker, path)
with extract(path, timeout=self.timeout,
extract_dir=self.tmp_dir) as extraction:
ctx = create_context(extraction.tmp_dir)
broker[ctx.__class__] = ctx
self.fire("on_extract", ctx, broker, extraction)
output = StringIO()
with self.Format(broker, stream=output):
dr.run(self.comps, broker=broker)
output.seek(0)
result = output.read()
self.fire("on_engine_success", broker, result)
return result
except Exception as ex:
self.fire("on_engine_failure", broker, ex)
raise
finally:
self.fire("on_engine_complete", broker)
def process_archives(archives):
for archive in archives:
with extract(archive) as ex:
ctx = create_context(ex.tmp_dir)
broker = dr.Broker()
broker[ctx.__class__] = ctx
yield dr.run(broker=broker)
def get_rule_hit_info(archive, rule_name, timeout=None, tmp_dir=None):
# We have to load everything again for multiprocessing or clustering to
# work. Even though imports are cached internally after the first call, we
# can still optimize a bit with this LOADED hack.
rule_func = dr.get_component(rule_name)
if not LOADED[0]:
load_default_plugins()
LOADED[0] = True
# this is also cached behind get_deps after the first call.
graph, bool_deps = get_deps(rule_func)
with extract(archive, timeout=timeout, extract_dir=tmp_dir) as arc:
ctx = create_context(arc.tmp_dir, None)
broker = dr.Broker()
broker[ctx.__class__] = ctx
results = dr.run(graph, broker=broker)
rule_result = results.get(rule_func)
rhr = results.get(RedHatRelease)
result = extract_hits(bool_deps, results)
result["archive"] = archive
result["key"] = rule_result.get_key() if rule_result else None
result["type"] = rule_result.__class__.__name__ if rule_result else None
result["make_fail"] = True if rule_result and isinstance(rule_result, make_response) else False
result["major"] = rhr.major if rhr else -1
result["minor"] = rhr.minor if rhr else -1
return result
def parse(self):
# pylint: disable=too-many-branches
"""Parse given archive."""
ARCHIVE_PARSE_COUNT.inc()
default_packages = ("insights.specs.default",
"insights.specs.insights_archive",
"insights.combiners", "insights.parsers")
for pkg in default_packages:
dr.load_components(pkg)
broker = dr.Broker()
with archives.extract(self.archive_path) as ex:
ctx = create_context(ex.tmp_dir, HostArchiveContext)
broker[ctx.__class__] = ctx
broker = dr.run(components=[Installed, DnfModules, YumReposD],
broker=broker)
if Installed in broker:
pkglist = broker[Installed]
self._delete_blacklisted_packages(pkglist.packages)
for pkg_name in pkglist.packages:
pkg = pkglist.get_max(pkg_name)
self.package_list.append("%s-%s:%s-%s.%s" %
(pkg.name, pkg.epoch, pkg.version,
pkg.release, pkg.arch))
else:
RPMDB_PARSE_FAILURE.inc()
LOGGER.error("Unable to parse package list from archive.")
return
if YumReposD in broker:
repolist = broker[YumReposD]
for repo_file in repolist:
if repo_file.file_name == 'redhat.repo':
for repo in repo_file:
if repo_file[repo].get(
'enabled',
'1').lower() in ('1', 'true', 'enabled',
'yes', 'on'):
self.repo_list.append(repo)
break
if not self.repo_list:
REPOLIST_PARSE_FAILURE.inc()
LOGGER.warning("Unable to parse RHSM repo list from archive.")
if DnfModules in broker:
for module in broker[DnfModules]:
for module_name in module.sections():
self.modules_list.append({
'module_name':
module_name,
'module_stream':
module.get(module_name, 'stream')
})
def _get_system_profile(archive_path):
profile = {}
default_packages = ("insights.specs.default",
"insights.specs.insights_archive",
"insights.combiners", "insights.parsers")
for pkg in default_packages:
dr.load_components(pkg)
broker = dr.Broker()
try:
with archives.extract(archive_path) as ex:
ctx = create_context(ex.tmp_dir, HostArchiveContext)
broker[ctx.__class__] = ctx
broker = dr.run(components=[
Specs.machine_id, Installed, DnfModules, YumReposD
],
broker=broker)
if Specs.machine_id in broker:
profile["id"] = broker[Specs.machine_id].content[0].strip()
profile["installed_packages"] = []
if Installed in broker:
pkglist = broker[Installed]
for pkg_name in pkglist.packages:
pkg = pkglist.get_max(pkg_name)
profile["installed_packages"].append(pkg.nevra)
profile["yum_repos"] = []
if YumReposD in broker:
repolist = broker[YumReposD]
for repo_file in repolist:
if repo_file.file_name == 'redhat.repo':
for repo in repo_file:
if repo_file[repo].get(
'enabled',
'1').lower() in ('1', 'true',
'enabled', 'yes',
'on'):
profile["yum_repos"].append(repo)
break
profile["dnf_modules"] = []
if DnfModules in broker:
for module in broker[DnfModules]:
for module_name in module.sections():
profile["dnf_modules"].append({
'name':
module_name,
'stream':
module.get(module_name, 'stream')
})
LOGGER.info(profile)
except InvalidContentType:
LOGGER.error("Unable to parse archive.")
return profile
def parse(self):
# pylint: disable=too-many-branches
"""Parse given archive."""
ARCHIVE_PARSE_COUNT.inc()
default_packages = ("insights.specs.default",
"insights.specs.insights_archive",
"insights.combiners", "insights.parsers")
for pkg in default_packages:
dr.load_components(pkg)
broker = dr.Broker()
with archives.extract(self.archive_path) as ex:
ctx = create_context(ex.tmp_dir, HostArchiveContext)
broker[ctx.__class__] = ctx
broker = dr.run(components=[
Installed, SubscriptionManagerReposListEnabled, DnfModules
],
broker=broker)
if Installed in broker:
pkglist = broker[Installed]
self._delete_blacklisted_packages(pkglist.packages)
for pkg_name in pkglist.packages:
pkg = pkglist.get_max(pkg_name)
self.package_list.append("%s-%s:%s-%s.%s" %
(pkg.name, pkg.epoch, pkg.version,
pkg.release, pkg.arch))
else:
RPMDB_PARSE_FAILURE.inc()
LOGGER.error("Unable to parse package list from archive.")
return
if SubscriptionManagerReposListEnabled in broker:
repolist = broker[SubscriptionManagerReposListEnabled]
for repo_record in repolist.records:
repo_label = repo_record.get("Repo ID", None)
if repo_label:
self.repo_list.append(repo_label)
if not self.repo_list:
REPOLIST_PARSE_FAILURE.inc()
LOGGER.warning("Unable to parse RHSM repo list from archive.")
if DnfModules in broker:
for module in broker[DnfModules]:
for module_name in module.sections():
self.modules_list.append({
'module_name':
module_name,
'module_stream':
module.get(module_name, 'stream')
})
def process_archives(graph, archives):
for archive in archives:
if os.path.isfile(archive):
with extract(archive) as ex:
ctx = create_context(ex.tmp_dir)
broker = dr.Broker()
broker[ctx.__class__] = ctx
yield dr.run(broker=broker)
else:
ctx = create_context(archive)
broker = dr.Broker()
broker[ctx.__class__] = ctx
yield dr.run(graph, broker=broker)
def upload(system_id):
user_agent = request.headers.get("User-Agent", "Unknown")
setattr(util.thread_context, "request_id",
request.headers.get("X-Request-Id", "Unknown"))
account_number = request.headers.get("X-Account", "")
file_size, file_loc = extract()
with archives.extract(file_loc) as ex:
results = handle(ex.tmp_dir, system_id, config=config)
response = handle_results(results, file_size, user_agent)
s3.save(file_loc, results["system"].get("system_id"), ex.content_type,
account_number)
shutil.rmtree(os.path.dirname(file_loc))
update_stats(results, user_agent)
return response
def analyze(paths, excludes=None):
if not isinstance(paths, list):
paths = [paths]
results = []
for path in paths:
if content_type.from_file(path) == "text/plain":
results.append(_load(path))
elif os.path.isdir(path):
results.extend(_process(path, excludes))
else:
with extract(path) as ex:
results.extend(_process(ex.tmp_dir, excludes))
return Result(children=results)
def process(self, path, broker=None):
broker = dr.Broker() if broker is None else broker
if os.path.isdir(path):
results = {"results": self._evaluate(broker, path)}
else:
with extract(path) as extraction:
results = {
"results": self._evaluate(broker, extraction.tmp_dir)
}
if self._include_timings:
results["timings"] = self._get_timings(broker)
if self._include_tracebacks:
results["tracebacks"] = self._get_tracebacks(broker)
return results
def _get_system_facts(archive_path):
facts = {}
default_packages = ("insights.specs.default",
"insights.specs.insights_archive",
"insights.combiners", "insights.parsers")
for pkg in default_packages:
dr.load_components(pkg)
broker = dr.Broker()
try:
with archives.extract(archive_path) as ex:
ctx = create_context(ex.tmp_dir, HostArchiveContext)
broker[ctx.__class__] = ctx
broker = dr.run(components=[Specs.machine_id], broker=broker)
if Specs.machine_id in broker:
facts["id"] = broker[Specs.machine_id].content[0].strip()
except InvalidContentType:
LOGGER.error("Unable to parse archive.")
return facts
def test_with_zip():
tmp_dir = tempfile.mkdtemp()
d = os.path.join(tmp_dir, 'sys', 'kernel')
os.makedirs(d)
with open(os.path.join(d, 'kexec_crash_size'), "w") as f:
f.write("ohyeahbaby")
try:
os.unlink("/tmp/test.zip")
except:
pass
# stolen from zipfile.py:main
def _add_to_zip(zf, path, zippath):
if os.path.isfile(path):
zf.write(path, zippath, zipfile.ZIP_DEFLATED)
elif os.path.isdir(path):
if zippath:
zf.write(path, zippath)
for nm in os.listdir(path):
_add_to_zip(zf, os.path.join(path, nm),
os.path.join(zippath, nm))
# else: ignore
with closing(zipfile.ZipFile("/tmp/test.zip", "w")) as zf:
_add_to_zip(zf, tmp_dir, os.path.basename(tmp_dir))
try:
with extract("/tmp/test.zip") as ex:
assert any(
f.endswith("/sys/kernel/kexec_crash_size")
for f in archives.get_all_files(ex.tmp_dir))
finally:
os.unlink("/tmp/test.zip")
subprocess.call(shlex.split("rm -rf %s" % tmp_dir))
def process(self, path, broker=None):
"""
Use this method to process archives. Pass a path to an archive and an
optional seed broker to use when evaluating it.
Archives are extracted outside the sandbox, and the temporary path is
sent inside for processing.
Arguments:
path (str): Path to an archive or directory to analyze.
Directories must be inside the temporary directory so the
sandbox can see them.
broker (insights.core.dr.Broker): Seed broker to use during the
evaluation. Observers registered with the seed broker won't
be fired since it is serialized into the sandbox before
evaluation.
"""
if os.path.isdir(path):
return self._runner_adapter_proxy.process(path, broker=broker)
else:
with extract(path) as extraction:
p = extraction.tmp_dir
return self._runner_adapter_proxy.process(p, broker=broker)
def process(self, path):
with extract(path) as ext:
for item in self.process_dir(ext.tmp_dir):
yield item
