def extraction(msg, extra, remove=True):
metrics.extraction_count.inc()
facts = {"system_profile": {}}
try:
with NamedTemporaryFile(delete=remove) as tf:
tf.write(get_archive(msg["url"]))
tf.flush()
logger.debug("extracting facts from %s", tf.name, extra=extra)
with extract(tf.name) as ex:
facts = get_canonical_facts(path=ex.tmp_dir)
facts["system_profile"] = get_system_profile(path=ex.tmp_dir)
except Exception as e:
logger.exception("Failed to extract facts: %s", str(e), extra=extra)
facts["error"] = str(e)
finally:
if facts["system_profile"].get("display_name"):
facts["display_name"] = facts["system_profile"].get("display_name")
if facts["system_profile"].get("satellite_id"):
facts["satellite_id"] = facts["system_profile"].get("satellite_id")
if facts["system_profile"].get("tags"):
facts["tags"] = facts["system_profile"].pop("tags")
groomed_facts = _remove_empties(_remove_bad_display_name(facts))
metrics.msg_processed.inc()
metrics.extract_success.inc()
return groomed_facts
def _create_new_broker(path=None):
"""
Create a broker and populate it by evaluating the path with all
registered datasources.
Args:
path (str): path to the archive or directory to analyze. ``None`` will
analyze the current system.
"""
datasources = dr.get_components_of_type(datasource)
def make_broker(ctx):
broker = dr.Broker()
broker[ctx.__class__] = ctx
if isinstance(ctx, SerializedArchiveContext):
h = Hydration(ctx.root)
broker = h.hydrate(broker=broker)
dr.run(datasources, broker=broker)
del broker[ctx.__class__]
return broker
if path:
if os.path.isdir(path):
ctx = create_context(path)
yield (path, make_broker(ctx))
else:
with extract(path) as e:
ctx = create_context(e.tmp_dir)
yield (e.tmp_dir, make_broker(ctx))
else:
yield (os.curdir, make_broker(HostContext()))
async def extract_facts(archive):
facts = {}
with extract(archive) as ex:
broker = run(root=ex.tmp_dir)
for k, v in CANONICAL_FACTS.items():
facts[k] = ('\n'.join(broker[v].content))
return facts
async def extract_facts(archive):
logger.info("extracting facts from %s", archive)
facts = {}
try:
with extract(archive) as ex:
facts = get_canonical_facts(path=ex.tmp_dir)
except (InvalidContentType, KeyError) as e:
facts['error'] = e.args[0]
return facts
def _download_and_extract_report(report_url):
download_response = requests.get(report_url)
if download_response.status_code != HTTPStatus.OK:
LOG.error("Unable to download the report. ERROR - %s",
download_response.reason)
else:
with NamedTemporaryFile() as tf:
tf.write(download_response.content)
tf.flush()
with extract(tf.name) as ex:
yield ex
def extract_facts(archive):
# TODO: facts, system_profiles, and errors are all passed through via the
# 'facts' hash. These should likely be split out.
logger.info("extracting facts from %s", archive)
facts = {}
try:
with extract(archive) as ex:
facts = get_canonical_facts(path=ex.tmp_dir)
facts['system_profile'] = get_system_profile(path=ex.tmp_dir)
except Exception as e:
logger.exception("Failed to extract facts")
facts['error'] = e
groomed_facts = _remove_empties(_remove_bad_display_name(facts))
return groomed_facts
def create_broker(root=None):
if not root:
broker = dr.Broker()
broker[HostContext] = HostContext()
yield broker
else:
def from_dir(d):
# ctx is returned here, but its already in the broker so not needed
_, broker = initialize_broker(d)
return broker
if os.path.isdir(root):
yield from_dir(root)
else:
with extract(root) as ex:
yield from_dir(ex.tmp_dir)
def create_broker(root=None):
if not root:
broker = dr.Broker()
broker[HostContext] = HostContext()
yield broker
else:
def from_dir(d):
broker = dr.Broker()
ctx = create_context(d, None)
broker[ctx.__class__] = ctx
return broker
if os.path.isdir(root):
yield from_dir(root)
else:
with extract(root) as ex:
yield from_dir(ex.tmp_dir)
def extraction(msg, extra, remove=True):
facts = {"system_profile": {}}
try:
with NamedTemporaryFile(delete=remove) as tf:
tf.write(get_archive(msg["url"]))
logger.debug("extracting facts from %s", tf.name, extra=extra)
with extract(tf.name) as ex:
facts = get_canonical_facts(path=ex.tmp_dir)
facts["system_profile"] = get_system_profile(path=ex.tmp_dir)
except Exception as e:
logger.exception("Failed to extract facts: %s", e, extra=extra)
facts["error"] = e
finally:
if facts["system_profile"].get("display_name"):
facts["display_name"] = facts["system_profile"].get("display_name")
groomed_facts = _remove_empties(_remove_bad_display_name(facts))
return groomed_facts
def _download_and_extract_report(report_url,
account_number,
custom_prefix=prefix):
download_response = requests.get(report_url)
LOG.info("%s - Downloading the report from %s.\n", custom_prefix,
report_url)
if download_response.status_code != HTTPStatus.OK:
archive_failed_to_download.labels(account_number=account_number).inc()
LOG.error("%s - Unable to download the report from %s. ERROR - %s\n",
custom_prefix, report_url, download_response.reason)
else:
archive_downloaded_success.labels(account_number=account_number).inc()
with NamedTemporaryFile() as tf:
tf.write(download_response.content)
LOG.debug("%s - Downloaded the report successfully from %s.\n",
custom_prefix, report_url)
tf.flush()
with extract(tf.name) as ex:
yield ex
def extract_facts(data, request_id, account, extra, remove=True):
# TODO: facts, system_profiles, and errors are all passed through via the
# 'facts' hash. These should likely be split out.
facts = {}
try:
with NamedTemporaryFile(delete=remove) as tf:
tf.write(data)
data = None
logger.info("extracting facts from %s", tf.name, extra=extra)
with extract(tf.name) as ex:
facts = get_canonical_facts(path=ex.tmp_dir)
facts['system_profile'] = get_system_profile(path=ex.tmp_dir)
except Exception as e:
logger.exception("Failed to extract facts: %s", e, extra=extra)
facts['error'] = e
else:
logger.info("Successfully extracted canonical facts", extra=extra)
finally:
if facts['system_profile'].get('display_name'):
facts['display_name'] = facts['system_profile'].get('display_name')
groomed_facts = _remove_empties(_remove_bad_display_name(facts))
return groomed_facts
canonical_facts = [
Specs.machine_id,
Specs.hostname,
Specs.redhat_release,
Specs.uname,
]
def get_archive_name():
import sys
import os
if len(sys.argv) < 2:
print("Need archive name")
sys.exit(1)
archive_name = sys.argv[1]
if not os.path.exists(archive_name):
print(f"Invalid archive path: {archive_name}")
sys.exit(1)
return archive_name
if __name__ == "__main__":
with extract(get_archive_name()) as ex:
broker = run(root=ex.tmp_dir)
for fact in canonical_facts:
print("\n".join(broker[fact].content))
