def extraction(msg, extra, remove=True): metrics.extraction_count.inc() facts = {"system_profile": {}} try: with NamedTemporaryFile(delete=remove) as tf: tf.write(get_archive(msg["url"])) tf.flush() logger.debug("extracting facts from %s", tf.name, extra=extra) with extract(tf.name) as ex: facts = get_canonical_facts(path=ex.tmp_dir) facts["system_profile"] = get_system_profile(path=ex.tmp_dir) except Exception as e: logger.exception("Failed to extract facts: %s", str(e), extra=extra) facts["error"] = str(e) finally: if facts["system_profile"].get("display_name"): facts["display_name"] = facts["system_profile"].get("display_name") if facts["system_profile"].get("satellite_id"): facts["satellite_id"] = facts["system_profile"].get("satellite_id") if facts["system_profile"].get("tags"): facts["tags"] = facts["system_profile"].pop("tags") groomed_facts = _remove_empties(_remove_bad_display_name(facts)) metrics.msg_processed.inc() metrics.extract_success.inc() return groomed_facts
def _create_new_broker(path=None): """ Create a broker and populate it by evaluating the path with all registered datasources. Args: path (str): path to the archive or directory to analyze. ``None`` will analyze the current system. """ datasources = dr.get_components_of_type(datasource) def make_broker(ctx): broker = dr.Broker() broker[ctx.__class__] = ctx if isinstance(ctx, SerializedArchiveContext): h = Hydration(ctx.root) broker = h.hydrate(broker=broker) dr.run(datasources, broker=broker) del broker[ctx.__class__] return broker if path: if os.path.isdir(path): ctx = create_context(path) yield (path, make_broker(ctx)) else: with extract(path) as e: ctx = create_context(e.tmp_dir) yield (e.tmp_dir, make_broker(ctx)) else: yield (os.curdir, make_broker(HostContext()))
async def extract_facts(archive): facts = {} with extract(archive) as ex: broker = run(root=ex.tmp_dir) for k, v in CANONICAL_FACTS.items(): facts[k] = ('\n'.join(broker[v].content)) return facts
async def extract_facts(archive): logger.info("extracting facts from %s", archive) facts = {} try: with extract(archive) as ex: facts = get_canonical_facts(path=ex.tmp_dir) except (InvalidContentType, KeyError) as e: facts['error'] = e.args[0] return facts
def _download_and_extract_report(report_url): download_response = requests.get(report_url) if download_response.status_code != HTTPStatus.OK: LOG.error("Unable to download the report. ERROR - %s", download_response.reason) else: with NamedTemporaryFile() as tf: tf.write(download_response.content) tf.flush() with extract(tf.name) as ex: yield ex
def extract_facts(archive): # TODO: facts, system_profiles, and errors are all passed through via the # 'facts' hash. These should likely be split out. logger.info("extracting facts from %s", archive) facts = {} try: with extract(archive) as ex: facts = get_canonical_facts(path=ex.tmp_dir) facts['system_profile'] = get_system_profile(path=ex.tmp_dir) except Exception as e: logger.exception("Failed to extract facts") facts['error'] = e groomed_facts = _remove_empties(_remove_bad_display_name(facts)) return groomed_facts
def create_broker(root=None): if not root: broker = dr.Broker() broker[HostContext] = HostContext() yield broker else: def from_dir(d): # ctx is returned here, but its already in the broker so not needed _, broker = initialize_broker(d) return broker if os.path.isdir(root): yield from_dir(root) else: with extract(root) as ex: yield from_dir(ex.tmp_dir)
def create_broker(root=None): if not root: broker = dr.Broker() broker[HostContext] = HostContext() yield broker else: def from_dir(d): broker = dr.Broker() ctx = create_context(d, None) broker[ctx.__class__] = ctx return broker if os.path.isdir(root): yield from_dir(root) else: with extract(root) as ex: yield from_dir(ex.tmp_dir)
def extraction(msg, extra, remove=True): facts = {"system_profile": {}} try: with NamedTemporaryFile(delete=remove) as tf: tf.write(get_archive(msg["url"])) logger.debug("extracting facts from %s", tf.name, extra=extra) with extract(tf.name) as ex: facts = get_canonical_facts(path=ex.tmp_dir) facts["system_profile"] = get_system_profile(path=ex.tmp_dir) except Exception as e: logger.exception("Failed to extract facts: %s", e, extra=extra) facts["error"] = e finally: if facts["system_profile"].get("display_name"): facts["display_name"] = facts["system_profile"].get("display_name") groomed_facts = _remove_empties(_remove_bad_display_name(facts)) return groomed_facts
def _download_and_extract_report(report_url, account_number, custom_prefix=prefix): download_response = requests.get(report_url) LOG.info("%s - Downloading the report from %s.\n", custom_prefix, report_url) if download_response.status_code != HTTPStatus.OK: archive_failed_to_download.labels(account_number=account_number).inc() LOG.error("%s - Unable to download the report from %s. ERROR - %s\n", custom_prefix, report_url, download_response.reason) else: archive_downloaded_success.labels(account_number=account_number).inc() with NamedTemporaryFile() as tf: tf.write(download_response.content) LOG.debug("%s - Downloaded the report successfully from %s.\n", custom_prefix, report_url) tf.flush() with extract(tf.name) as ex: yield ex
def extract_facts(data, request_id, account, extra, remove=True): # TODO: facts, system_profiles, and errors are all passed through via the # 'facts' hash. These should likely be split out. facts = {} try: with NamedTemporaryFile(delete=remove) as tf: tf.write(data) data = None logger.info("extracting facts from %s", tf.name, extra=extra) with extract(tf.name) as ex: facts = get_canonical_facts(path=ex.tmp_dir) facts['system_profile'] = get_system_profile(path=ex.tmp_dir) except Exception as e: logger.exception("Failed to extract facts: %s", e, extra=extra) facts['error'] = e else: logger.info("Successfully extracted canonical facts", extra=extra) finally: if facts['system_profile'].get('display_name'): facts['display_name'] = facts['system_profile'].get('display_name') groomed_facts = _remove_empties(_remove_bad_display_name(facts)) return groomed_facts
canonical_facts = [ Specs.machine_id, Specs.hostname, Specs.redhat_release, Specs.uname, ] def get_archive_name(): import sys import os if len(sys.argv) < 2: print("Need archive name") sys.exit(1) archive_name = sys.argv[1] if not os.path.exists(archive_name): print(f"Invalid archive path: {archive_name}") sys.exit(1) return archive_name if __name__ == "__main__": with extract(get_archive_name()) as ex: broker = run(root=ex.tmp_dir) for fact in canonical_facts: print("\n".join(broker[fact].content))