def verify_connectivity(): """ Verify connectivity to satellite server """ logger.debug("Verifying Connectivity") for item, value in InsightsClient.config.items(APP_NAME): if item != 'password' and item != 'proxy' and item != 'systemid': logger.debug("%s:%s", item, value) ic = InsightsConnection() try: branch_info = ic.branch_info() except requests.ConnectionError as e: logger.debug(e) logger.debug("Failed to connect to satellite") return False except LookupError as e: logger.debug(e) logger.debug("Failed to parse response from satellite") return False try: remote_leaf = branch_info['remote_leaf'] return remote_leaf except LookupError as e: logger.debug(e) logger.debug("Failed to find accurate branch_info") return False
def registration_check(self): # check local registration record if os.path.isfile(constants.registered_file): local_record = 'System is registered.' with open(constants.registered_file) as reg_file: local_record += ' Registered at ' + reg_file.readline() else: local_record = 'System is NOT registered.' if os.path.isfile(constants.unregistered_file): with open(constants.unregistered_file) as reg_file: local_record += ' Unregistered at ' + reg_file.readline() pconn = InsightsConnection(self.config) api_reg_status = pconn.api_registration_check() if type(api_reg_status) is bool: if api_reg_status: api_record = 'Insights API confirms registration.' else: api_record = 'Insights API could not be reached to confirm registration status.' elif api_reg_status is None: api_record = 'Insights API says this machine is NOT registered.' else: api_record = 'Insights API says this machine was unregistered at ' + api_reg_status return [local_record, api_record]
def register(): """ Do registration using basic auth """ username = InsightsClient.config.get(APP_NAME, 'username') password = InsightsClient.config.get(APP_NAME, 'password') authmethod = InsightsClient.config.get(APP_NAME, 'authmethod') # TODO validate this is boolean somewhere in config load auto_config = InsightsClient.config.getboolean(APP_NAME, 'auto_config') if not username and not password and not auto_config and authmethod == 'BASIC': print 'Please enter your Red Hat Customer Portal Credentials' sys.stdout.write('Username: '******'Would you like to save these credentials? (y/n) ') save = raw_input().strip() InsightsClient.config.set(APP_NAME, 'username', username) InsightsClient.config.set(APP_NAME, 'password', password) logger.debug('savestr: %s', save) if save.lower() == 'y' or save.lower() == 'yes': logger.debug('Writing user/pass to config') cmd = ('/bin/sed -e \'s/^username.*=.*$/username='******'/\' ' + '-e \'s/^password.*=.*$/password='******'/\' ' + constants.default_conf_file) status = DataCollector().run_command_get_output(cmd, nolog=True) with open(constants.default_conf_file, 'w') as config_file: config_file.write(status['output']) config_file.flush() pconn = InsightsConnection() return pconn.register()
def register(config, group_id=None): """ Do registration using basic auth """ username = config.get(APP_NAME, 'username') password = config.get(APP_NAME, 'password') if (( username == "" and password == "" and config.get(APP_NAME, 'authmethod') == 'BASIC') and not config.get(APP_NAME, 'auto_config')): # Get input from user print "Please enter your Red Hat Customer Portal Credentials" sys.stdout.write('User Name: ') username = raw_input().strip() password = getpass.getpass() sys.stdout.write("Would you like to save these credentials? (y/n) ") save = raw_input().strip() config.set(APP_NAME, 'username', username) config.set(APP_NAME, 'password', password) logger.debug("savestr: %s", save) if save.lower() == "y" or save.lower() == "yes": logger.debug("writing user/pass to config file") cmd = ("/bin/sed -e 's/^username.*=.*$/username="******"/' " + "-e 's/^password.*=.*$/password="******"/' " + constants.default_conf_file) status = DataCollector().run_command_get_output(cmd, nolog=True) config_file = open(constants.default_conf_file, 'w') config_file.write(status['output']) config_file.flush() pconn = InsightsConnection(config) return pconn.register(group_id)
def register(config, group_id=None): """ Do registration using basic auth """ username = config.get(APP_NAME, 'username') password = config.get(APP_NAME, 'password') if (((username == "") and (password == "") and (config.get(APP_NAME, 'authmethod') == 'BASIC')) and not (config.get(APP_NAME, 'auto_config'))): # Get input from user print "Please enter your Red Hat Customer Portal Credentials" sys.stdout.write('User Name: ') username = raw_input().strip() password = getpass.getpass() sys.stdout.write("Would you like to save these credentials? (y/n) ") save = raw_input().strip() config.set(APP_NAME, 'username', username) config.set(APP_NAME, 'password', password) logger.debug("savestr: %s", save) if save.lower() == "y" or save.lower() == "yes": logger.debug("writing user/pass to config file") cmd = ("/bin/sed -e 's/^username.*=.*$/username="******"/' " + "-e 's/^password.*=.*$/password="******"/' " + constants.default_conf_file) status = DataCollector().run_command_get_output(cmd, nolog=True) config_file = open(constants.default_conf_file, 'w') config_file.write(status['output']) config_file.flush() pconn = InsightsConnection(config) return pconn.register(group_id)
def verify_connectivity(config): """ Verify connectivity to satellite server """ logger.debug("Verifying Connectivity") for item, value in config.items(APP_NAME): if item != 'password' and item != 'proxy' and item != 'systemid': logger.debug("%s:%s", item, value) ic = InsightsConnection(config) try: branch_info = ic.branch_info() except requests.ConnectionError as e: logger.debug(e) logger.debug("Failed to connect to satellite") return False except LookupError as e: logger.debug(e) logger.debug("Failed to parse response from satellite") return False try: remote_leaf = branch_info['remote_leaf'] return remote_leaf except LookupError as e: logger.debug(e) logger.debug("Failed to find accurate branch_info") return False
def verify_connectivity(): """ Verify connectivity to satellite server """ logger.debug("Verifying Connectivity") ic = InsightsConnection() try: branch_info = ic.branch_info() except requests.ConnectionError as e: logger.debug(e) logger.debug("Failed to connect to satellite") return False except LookupError as e: logger.debug(e) logger.debug("Failed to parse response from satellite") return False try: remote_leaf = branch_info['remote_leaf'] return remote_leaf except LookupError as e: logger.debug(e) logger.debug("Failed to find accurate branch_info") return False
def register(): """ Do registration using basic auth """ username = InsightsClient.config.get(APP_NAME, 'username') password = InsightsClient.config.get(APP_NAME, 'password') authmethod = InsightsClient.config.get(APP_NAME, 'authmethod') # TODO validate this is boolean somewhere in config load auto_config = InsightsClient.config.getboolean(APP_NAME, 'auto_config') if not username and not password and not auto_config and authmethod == 'BASIC': print 'Please enter your Red Hat Customer Portal Credentials' sys.stdout.write('Username: '******'Would you like to save these credentials? (y/n) ') save = raw_input().strip() InsightsClient.config.set(APP_NAME, 'username', username) InsightsClient.config.set(APP_NAME, 'password', password) logger.debug('savestr: %s', save) if save.lower() == 'y' or save.lower() == 'yes': logger.debug('Writing user/pass to config') modify_config_file({'username': username, 'password': password}) pconn = InsightsConnection() return pconn.register()
def collect_data_and_upload(config, options): """ All the heavy lifting done here """ pconn = InsightsConnection(config) pconn.check_registration() branch_info = pconn.branch_info() pc = InsightsConfig(config, pconn) dc = DataCollector() logger.info('Collecting Insights data') start = time.clock() dynamic_config = pc.get_conf(options.update) elapsed = (time.clock() - start) logger.debug("Dynamic Config Elapsed Time: %s", elapsed) start = time.clock() dc.run_commands(dynamic_config) elapsed = (time.clock() - start) logger.debug("Command Collection Elapsed Time: %s", elapsed) start = time.clock() dc.copy_files(dynamic_config) elapsed = (time.clock() - start) logger.debug("File Collection Elapsed Time: %s", elapsed) dc.write_branch_info(branch_info) obfuscate = config.getboolean(APP_NAME, "obfuscate") if not options.no_tar_file: tar_file = dc.done(config) if not options.no_upload: logger.info('Uploading Insights data,' ' this may take a few minutes') pconn.upload_archive(tar_file) logger.info( 'Check https://access.redhat.com/labs/insights in an hour') if not obfuscate and not options.keep_archive: dc.archive.delete_tmp_dir() else: if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(tar_file)) else: logger.info('Insights data retained in %s', tar_file) else: logger.info('See Insights data in %s', tar_file) else: logger.info('See Insights data in %s', dc.archive.archive_dir)
def handle_startup(): """ Handle startup options """ # ----do X and exit options---- # show version and exit if InsightsClient.options.version: print constants.version sys.exit() if (InsightsClient.options.container_mode and not InsightsClient.options.run_here and insights_client_container_is_available()): sys.exit(run_in_container()) if (InsightsClient.options.container_mode and not InsightsClient.options.only): logger.error( "Client running in container mode but no image/container specified via --only." ) sys.exit(1) if InsightsClient.options.only != None and len( InsightsClient.options.only) < 12: logger.error( "Image/Container ID must be atleast twelve characters long.") sys.exit(1) if InsightsClient.options.validate: validate_remove_file() sys.exit() if InsightsClient.options.enable_schedule and InsightsClient.options.disable_schedule: logger.error( 'Conflicting options: --enable-schedule and --disable-schedule') sys.exit(1) if InsightsClient.options.enable_schedule: # enable automatic scheduling InsightsSchedule() InsightsClient.config.set(APP_NAME, 'no_schedule', False) logger.info('Automatic scheduling for Insights has been enabled.') logger.debug('Updating config...') modify_config_file({'no_schedule': 'False'}) sys.exit() if InsightsClient.options.disable_schedule: # disable automatic schedling InsightsSchedule(set_cron=False).remove_scheduling() InsightsClient.config.set(APP_NAME, 'no_schedule', True) logger.info('Automatic scheduling for Insights has been disabled.') logger.debug('Updating config...') modify_config_file({'no_schedule': 'True'}) sys.exit() # do auto_config here, for connection-related 'do X and exit' options if InsightsClient.config.getboolean( APP_NAME, 'auto_config') and not InsightsClient.options.offline: # Try to discover if we are connected to a satellite or not try_auto_configuration() if InsightsClient.options.test_connection: pconn = InsightsConnection() rc = pconn.test_connection() sys.exit(rc) if InsightsClient.options.status: reg_check = registration_check() logger.info('\n'.join(reg_check['messages'])) # exit with !status, 0 for True, 1 for False sys.exit(not reg_check['status']) if InsightsClient.options.support: support = InsightsSupport() support.collect_support_info() sys.exit() # ----config options---- # log the config # ignore password and proxy -- proxy might have pw for item, value in InsightsClient.config.items(APP_NAME): if item != 'password' and item != 'proxy': logger.debug("%s:%s", item, value) if InsightsClient.config.getboolean( APP_NAME, 'auto_update') and not InsightsClient.options.offline: # TODO: config updates option, but in GPG option, the option updates # the config. make this consistent InsightsClient.options.update = True # disable automatic scheduling if it was set in the config, and if the job exists if InsightsClient.config.getboolean(APP_NAME, 'no_schedule'): cron = InsightsSchedule(set_cron=False) if cron.already_linked(): cron.remove_scheduling() logger.debug( 'Automatic scheduling for Insights has been disabled.') # ----modifier options---- if InsightsClient.options.no_gpg: logger.warn("WARNING: GPG VERIFICATION DISABLED") InsightsClient.config.set(APP_NAME, 'gpg', 'False') if InsightsClient.options.just_upload: if InsightsClient.options.offline or InsightsClient.options.no_upload: logger.error( 'Cannot use --just-upload in combination with --offline or --no-upload.' ) sys.exit(1) # override these for great justice InsightsClient.options.no_tar_file = False InsightsClient.options.keep_archive = True # if InsightsClient.options.container_mode and InsightsClient.options.no_tar_file: # logger.error('Invalid combination: --container and --no-tar-file') # sys.exit(1) # can't use bofa if InsightsClient.options.from_stdin and InsightsClient.options.from_file: logger.error('Can\'t use both --from-stdin and --from-file.') sys.exit(1) # handle some docker/atomic flags if InsightsClient.options.use_docker and InsightsClient.options.use_atomic: logger.error('Cant\'t use both --use-docker and --use-atomic.') sys.exit(1) if InsightsClient.options.to_stdout: InsightsClient.options.no_upload = True # ----register options---- # put this first to avoid conflicts with register if InsightsClient.options.unregister: pconn = InsightsConnection() pconn.unregister() sys.exit() # force-reregister -- remove machine-id files and registration files # before trying to register again new = False if InsightsClient.options.reregister: new = True InsightsClient.options.register = True delete_registered_file() delete_unregistered_file() delete_machine_id() logger.debug('Machine-id: %s', generate_machine_id(new)) if InsightsClient.options.register: try_register() if not InsightsClient.config.getboolean(APP_NAME, 'no_schedule'): InsightsSchedule() # check registration before doing any uploads # Ignore if in offline mode if not InsightsClient.options.register and not InsightsClient.options.offline: msg, is_registered = _is_client_registered() if not is_registered: logger.error(msg) sys.exit(1)
def _main(): """ Main entry point Parse cmdline options Parse config file Call data collector """ global logger sys.excepthook = handle_exception parser = optparse.OptionParser() set_up_options(parser) options, args = parser.parse_args() if len(args) > 0: parser.error("Unknown arguments: %s" % args) sys.exit(1) if options.version: print constants.version sys.exit() if options.validate: validate_remove_file() sys.exit() config = parse_config_file() logger, handler = set_up_logging(config, options) # Defer logging till it's ready logger.debug('invoked with args: %s', options) logger.debug("Version: " + constants.version) # Generate /etc/machine-id if it does not exist new = False if options.reregister: new = True options.register = True logger.debug("Machine-ID: " + generate_machine_id(new)) # Disable GPG verification if options.no_gpg: logger.warn("WARNING: GPG VERIFICATION DISABLED") config.set(APP_NAME, 'gpg', 'False') # Log config except the password # and proxy as it might have a pw as well for item, value in config.items(APP_NAME): if item != 'password' and item != 'proxy': logger.debug("%s:%s", item, value) if config.getboolean(APP_NAME, 'auto_update'): options.update = True if config.getboolean(APP_NAME, 'auto_config'): # Try to discover if we are connected to a satellite or not try_auto_configuration(config) # Set the schedule InsightsSchedule(options) # Test connection, useful for proxy debug if options.test_connection: pconn = InsightsConnection(config) pconn.test_connection() if options.unregister: pconn = InsightsConnection(config) pconn.unregister() sys.exit() # Handle registration, grouping, and display name if options.register: opt_group = options.group hostname, opt_group, display_name = register(config, options) if options.display_name is None and options.group is None: logger.info('Successfully registered %s', hostname) elif options.display_name is None: logger.info('Successfully registered %s in group %s', hostname, opt_group) else: logger.info('Successfully registered %s as %s in group %s', hostname, display_name, opt_group) # Check for .unregistered file if os.path.isfile(constants.unregistered_file): logger.error("This machine has been unregistered") logger.error("Use --register if you would like to re-register this machine") logger.error("Exiting") sys.exit(1) # do work collect_data_and_upload(config, options) handler.doRollover()
def collect_data_and_upload(config, options): """ All the heavy lifting done here """ pconn = InsightsConnection(config) try: branch_info = pconn.branch_info() except requests.ConnectionError: logger.error("ERROR: Could not connect to determine branch information") sys.exit() except LookupError: logger.error("ERROR: Could not determine branch information") sys.exit() pc = InsightsConfig(config, pconn) dc = DataCollector() start = time.clock() collection_rules, rm_conf = pc.get_conf(options.update) elapsed = (time.clock() - start) logger.debug("Collection Rules Elapsed Time: %s", elapsed) start = time.clock() logger.info('Starting to collect Insights data') dc.run_commands(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("Command Collection Elapsed Time: %s", elapsed) start = time.clock() dc.copy_files(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("File Collection Elapsed Time: %s", elapsed) dc.write_branch_info(branch_info) obfuscate = config.getboolean(APP_NAME, "obfuscate") if not options.no_tar_file: tar_file = dc.done(config, rm_conf) if not options.no_upload: logger.info('Uploading Insights data,' ' this may take a few minutes') for tries in range(options.retries): status = pconn.upload_archive(tar_file) if status == 201: logger.info("Upload completed successfully!") break else: logger.error("Upload attempt %d of %d failed! Status Code: %s", tries+1, options.retries, status) if tries +1 != options.retries: logger.info("Waiting %d seconds then retrying", constants.sleep_time) time.sleep(constants.sleep_time) else: logger.error("All attempts to upload have failed!") logger.error("Please see %s for additional information", constants.default_log_file) if not obfuscate and not options.keep_archive: dc.archive.delete_tmp_dir() else: if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(tar_file)) else: logger.info('Insights data retained in %s', tar_file) else: logger.info('See Insights data in %s', tar_file) else: logger.info('See Insights data in %s', dc.archive.archive_dir)
def _support_diag_dump(self): ''' Collect log info for debug ''' # check insights config cfg_block = [] pconn = InsightsConnection() logger.info('Insights version: %s', get_nvr()) reg_check = registration_check(pconn) cfg_block.append('Registration check:') for key in reg_check: cfg_block.append(key + ': ' + str(reg_check[key])) lastupload = 'never' if os.path.isfile(constants.lastupload_file): with open(constants.lastupload_file) as upl_file: lastupload = upl_file.readline().strip() cfg_block.append('\nLast successful upload was ' + lastupload) cfg_block.append('auto_config: ' + str(config['auto_config'])) if config['proxy']: obfuscated_proxy = re.sub(r'(.*)(:)(.*)(@.*)', r'\1\2********\4', config['proxy']) else: obfuscated_proxy = 'None' cfg_block.append('proxy: ' + obfuscated_proxy) logger.info('\n'.join(cfg_block)) logger.info('python-requests: %s', requests.__version__) succ = pconn.test_connection() if succ == 0: logger.info('Connection test: PASS\n') else: logger.info('Connection test: FAIL\n') # run commands commands = [ 'uname -a', 'cat /etc/redhat-release', 'env', 'sestatus', 'subscription-manager identity', 'systemctl cat insights-client.timer', 'systemctl cat insights-client.service', 'systemctl status insights-client.timer', 'systemctl status insights-client.service' ] for cmd in commands: logger.info("Running command: %s", cmd) try: proc = Popen(shlex.split(cmd), shell=False, stdout=PIPE, stderr=STDOUT, close_fds=True) stdout, stderr = proc.communicate() except OSError as o: if 'systemctl' not in cmd: # suppress output for systemctl cmd failures logger.info('Error running command "%s": %s', cmd, o) except Exception as e: # unknown error logger.info("Process failed: %s", e) logger.info("Process output: \n%s", stdout) # check available disk space for /var/tmp tmp_dir = '/var/tmp' dest_dir_stat = os.statvfs(tmp_dir) dest_dir_size = (dest_dir_stat.f_bavail * dest_dir_stat.f_frsize) logger.info( 'Available space in %s:\t%s bytes\t%.1f 1K-blocks\t%.1f MB', tmp_dir, dest_dir_size, dest_dir_size / 1024.0, (dest_dir_size / 1024.0) / 1024.0)
def collect_data_and_upload(rc=0): """ All the heavy lifting done here Run through "targets" - could be just one (host, default) or many (containers+host) """ # initialize collection targets # for now we do either containers OR host -- not both at same time if InsightsClient.options.container_mode: targets = get_targets() targets = targets + constants.default_target else: targets = constants.default_target if InsightsClient.options.offline: logger.warning("Assuming remote branch and leaf value of -1") pconn = None branch_info = constants.default_branch_info else: pconn = InsightsConnection() # TODO: change these err msgs to be more meaningful , i.e. # "could not determine login information" if pconn: try: branch_info = pconn.branch_info() except requests.ConnectionError: branch_info = handle_branch_info_error( "Could not connect to determine branch information") except LookupError: branch_info = handle_branch_info_error( "Could not determine branch information") pc = InsightsConfig(pconn) if InsightsClient.options.just_upload: if not os.path.exists(InsightsClient.options.just_upload): logger.error('No file %s', InsightsClient.options.just_upload) return 1 tar_file = InsightsClient.options.just_upload rc = _do_upload(pconn, tar_file, 'dummy', 0) return rc # load config from stdin/file if specified try: stdin_config = {} if InsightsClient.options.from_file: with open(InsightsClient.options.from_file, 'r') as f: stdin_config = json.load(f) elif InsightsClient.options.from_stdin: stdin_config = json.load(sys.stdin) if ((InsightsClient.options.from_file or InsightsClient.options.from_stdin) and ('uploader.json' not in stdin_config or 'sig' not in stdin_config)): raise ValueError except: logger.error('ERROR: Invalid config for %s! Exiting...', ('--from-file' if InsightsClient.options.from_file else '--from-stdin')) sys.exit(1) start = time.clock() collection_rules, rm_conf = pc.get_conf(InsightsClient.options.update, stdin_config) collection_elapsed = (time.clock() - start) logger.debug("Rules configuration loaded. Elapsed time: %s", collection_elapsed) individual_archives = [] for t in targets: # defaults archive = None container_connection = None mp = None obfuscate = None # archive metadata archive_meta = {} try: if t['type'] == 'docker_image': container_connection = open_image(t['name']) logging_name = 'Docker image ' + t['name'] archive_meta['display_name'] = get_repotag(t['name']) archive_meta['docker_id'] = t['name'] if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) continue elif t['type'] == 'docker_container': container_connection = open_container(t['name']) logging_name = 'Docker container ' + t['name'] archive_meta['display_name'] = t['name'] if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) continue elif t['type'] == 'host': logging_name = determine_hostname() archive_meta['display_name'] = determine_hostname(InsightsClient.options.display_name) else: logger.error('Unexpected analysis target: %s', t['type']) continue archive_meta['type'] = t['type'].replace('docker_', '') archive_meta['product'] = 'Docker' archive_meta['system_id'] = generate_analysis_target_id(t['type'], t['name']) collection_start = time.clock() archive = InsightsArchive(compressor=InsightsClient.options.compressor if not InsightsClient.options.container_mode else "none", target_name=t['name']) atexit.register(_delete_archive, archive) dc = DataCollector(archive, mountpoint=mp, target_name=t['name'], target_type=t['type']) logger.info('Starting to collect Insights data for %s', logging_name) dc.run_collection(collection_rules, rm_conf, branch_info) elapsed = (time.clock() - start) logger.debug("Data collection complete. Elapsed time: %s", elapsed) obfuscate = InsightsClient.config.getboolean(APP_NAME, "obfuscate") # include rule refresh time in the duration collection_duration = (time.clock() - collection_start) + collection_elapsed if InsightsClient.options.no_tar_file: logger.info('See Insights data in %s', dc.archive.archive_dir) return rc tar_file = dc.done(collection_rules, rm_conf) # add archives to list of individual uploads archive_meta['tar_file'] = tar_file individual_archives.append(archive_meta) finally: # called on loop iter end or unexpected exit if container_connection: container_connection.close() # if multiple targets (container mode), add all archives to single archive if InsightsClient.options.container_mode: full_archive = InsightsArchive(compressor=InsightsClient.options.compressor) for a in individual_archives: shutil.copy(a['tar_file'], full_archive.archive_dir) # don't want insights_commands in meta archive shutil.rmtree(full_archive.cmd_dir) metadata = _create_metadata_json(individual_archives) full_archive.add_metadata_to_archive(json.dumps(metadata), 'metadata.json') full_tar_file = full_archive.create_tar_file(full_archive=True) # if only one target (regular mode), just upload one else: full_archive = archive full_tar_file = tar_file if InsightsClient.options.offline or InsightsClient.options.no_upload: handle_file_output(full_tar_file, full_archive) return rc # do the upload rc = _do_upload(pconn, full_tar_file, logging_name, collection_duration) if InsightsClient.options.keep_archive: logger.info('Insights data retained in %s', full_tar_file) return rc if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(full_tar_file)) full_archive.delete_archive_dir() return rc
def collect_data_and_upload(rc=0): """ All the heavy lifting done here Run through "targets" - could be just ONE (host, default) or ONE (container/image) """ # initialize collection targets # for now we do either containers OR host -- not both at same time if InsightsClient.options.container_mode: logger.debug("Client running in container/image mode.") logger.debug("Scanning for matching container/image.") targets = get_targets() else: logger.debug("Host selected as scanning target.") targets = constants.default_target # if there are no targets to scan then bail if not len(targets): logger.debug("No targets were found. Exiting.") sys.exit(1) if InsightsClient.options.offline: logger.warning("Assuming remote branch and leaf value of -1") pconn = None branch_info = constants.default_branch_info else: pconn = InsightsConnection() # TODO: change these err msgs to be more meaningful , i.e. # "could not determine login information" if pconn: try: branch_info = pconn.branch_info() except requests.ConnectionError: branch_info = handle_branch_info_error( "Could not connect to determine branch information") except LookupError: branch_info = handle_branch_info_error( "Could not determine branch information") pc = InsightsConfig(pconn) tar_file = None if InsightsClient.options.just_upload: if not os.path.exists(InsightsClient.options.just_upload): logger.error('No file %s', InsightsClient.options.just_upload) return 1 tar_file = InsightsClient.options.just_upload rc = _do_upload(pconn, tar_file, 'dummy', 0) return rc # load config from stdin/file if specified try: stdin_config = {} if InsightsClient.options.from_file: with open(InsightsClient.options.from_file, 'r') as f: stdin_config = json.load(f) elif InsightsClient.options.from_stdin: stdin_config = json.load(sys.stdin) if ((InsightsClient.options.from_file or InsightsClient.options.from_stdin) and ('uploader.json' not in stdin_config or 'sig' not in stdin_config)): raise ValueError if ((InsightsClient.options.from_file or InsightsClient.options.from_stdin) and 'branch_info' in stdin_config and stdin_config['branch_info'] is not None): branch_info = stdin_config['branch_info'] except: logger.error('ERROR: Invalid config for %s! Exiting...', ('--from-file' if InsightsClient.options.from_file else '--from-stdin')) sys.exit(1) start = time.clock() collection_rules, rm_conf = pc.get_conf(InsightsClient.options.update, stdin_config) collection_elapsed = (time.clock() - start) logger.debug("Rules configuration loaded. Elapsed time: %s", collection_elapsed) individual_archives = [] for t in targets: # defaults archive = None container_connection = None mp = None obfuscate = None # archive metadata archive_meta = {} try: if t['type'] == 'docker_image': container_connection = open_image(t['name']) logging_name = 'Docker image ' + t['name'] archive_meta['docker_id'] = t['name'] archive_meta['display_name'] = docker_display_name( t['name'], t['type'].replace('docker_', '')) logger.debug('Docker display_name: %s', archive_meta['display_name']) logger.debug('Docker docker_id: %s', archive_meta['docker_id']) if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) sys.exit(1) elif t['type'] == 'docker_container': container_connection = open_container(t['name']) logging_name = 'Docker container ' + t['name'] archive_meta['docker_id'] = t['name'] archive_meta['display_name'] = docker_display_name( t['name'], t['type'].replace('docker_', '')) logger.debug('Docker display_name: %s', archive_meta['display_name']) logger.debug('Docker docker_id: %s', archive_meta['docker_id']) if container_connection: mp = container_connection.get_fs() else: logger.error('Could not open %s for analysis', logging_name) sys.exit(1) elif t['type'] == 'host': logging_name = determine_hostname() archive_meta['display_name'] = determine_hostname( InsightsClient.options.display_name) else: logger.error('Unexpected analysis target: %s', t['type']) sys.exit(1) archive_meta['type'] = t['type'].replace('docker_', '') archive_meta['product'] = 'Docker' archive_meta['system_id'] = generate_analysis_target_id( t['type'], t['name']) collection_start = time.clock() archive = InsightsArchive( compressor=InsightsClient.options.compressor if not InsightsClient.options.container_mode else "none", target_name=t['name']) atexit.register(_delete_archive, archive) dc = DataCollector(archive, InsightsClient.config, mountpoint=mp, target_name=t['name'], target_type=t['type']) logger.info('Starting to collect Insights data for %s', logging_name) dc.run_collection(collection_rules, rm_conf, branch_info) elapsed = (time.clock() - start) logger.debug("Data collection complete. Elapsed time: %s", elapsed) obfuscate = InsightsClient.config.getboolean(APP_NAME, "obfuscate") # include rule refresh time in the duration collection_duration = (time.clock() - collection_start) + collection_elapsed # add custom metadata about a host if provided by from_file # use in the OSE case if InsightsClient.options.from_file: with open(InsightsClient.options.from_file, 'r') as f: stdin_config = json.load(f) if 'metadata' in stdin_config: archive.add_metadata_to_archive( json.dumps(stdin_config['metadata']), 'metadata.json') if InsightsClient.options.no_tar_file: logger.info('See Insights data in %s', dc.archive.archive_dir) return rc tar_file = dc.done(collection_rules, rm_conf) # add archives to list of individual uploads archive_meta['tar_file'] = tar_file individual_archives.append(archive_meta) finally: # called on loop iter end or unexpected exit if container_connection: container_connection.close() # if multiple targets (container mode), add all archives to single archive # if InsightsClient.options.container_mode: if False: # we only run single collections now (not the uber archives), bypass this full_archive = InsightsArchive( compressor=InsightsClient.options.compressor) for a in individual_archives: shutil.copy(a['tar_file'], full_archive.archive_dir) # don't want insights_commands in meta archive shutil.rmtree(full_archive.cmd_dir) metadata = _create_metadata_json(individual_archives) full_archive.add_metadata_to_archive(json.dumps(metadata), 'metadata.json') full_tar_file = full_archive.create_tar_file(full_archive=True) # if only one target (regular mode), just upload one else: full_archive = archive full_tar_file = tar_file if InsightsClient.options.offline or InsightsClient.options.no_upload: handle_file_output(full_tar_file, full_archive) return rc # do the upload rc = _do_upload(pconn, full_tar_file, logging_name, collection_duration) if InsightsClient.options.keep_archive: logger.info('Insights data retained in %s', full_tar_file) return rc if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(full_tar_file)) full_archive.delete_archive_dir() return rc
def handle_startup(options, config): """ Handle startup options """ if options.version: print constants.version sys.exit() if options.validate: validate_remove_file() sys.exit() # Generate /etc/machine-id if it does not exist new = False if options.reregister: new = True options.register = True logger.debug("Machine-ID: " + generate_machine_id(new)) # Disable GPG verification if options.no_gpg: logger.warn("WARNING: GPG VERIFICATION DISABLED") config.set(APP_NAME, 'gpg', 'False') # Log config except the password # and proxy as it might have a pw as well for item, value in config.items(APP_NAME): if item != 'password' and item != 'proxy': logger.debug("%s:%s", item, value) if config.getboolean(APP_NAME, 'auto_update'): options.update = True if config.getboolean(APP_NAME, 'auto_config'): # Try to discover if we are connected to a satellite or not try_auto_configuration(config) # Set the schedule InsightsSchedule() # Test connection, useful for proxy debug if options.test_connection: pconn = InsightsConnection(config) pconn.test_connection() if options.unregister: pconn = InsightsConnection(config) pconn.unregister() sys.exit() # Handle registration, grouping, and display name if options.register: opt_group = options.group message, hostname, opt_group, display_name = register(config, options) if options.display_name is None and options.group is None: logger.info('Successfully registered %s', hostname) elif options.display_name is None: logger.info('Successfully registered %s in group %s', hostname, opt_group) else: logger.info('Successfully registered %s as %s in group %s', hostname, display_name, opt_group) logger.info(message) # Check for .unregistered file if os.path.isfile(constants.unregistered_file): logger.error("This machine has been unregistered") logger.error("Use --register if you would like to re-register this machine") logger.error("Exiting") sys.exit(1)
def _main(): """ Main entry point Parse cmdline options Parse config file Call data collector """ global logger sys.excepthook = handle_exception parser = optparse.OptionParser() set_up_options(parser) options, args = parser.parse_args() if len(args) > 0: parser.error("Unknown arguments: %s" % args) sys.exit(1) if options.satellite_group and not options.register: parser.error("--satellite-group must be used with --register") if options.version: print constants.version sys.exit() if options.daily and options.weekly: parser.error("options --daily and --weekly are mutually exclusive") config = parse_config_file() logger, handler = set_up_logging(config, options.verbose) # Defer logging till it's ready logger.debug('invoked with args: %s', options) logger.debug("Version: " + constants.version) # Generate /etc/machine-id if it does not exist new = False if options.regenerate: new = True logger.debug("Machine-ID: " + generate_machine_id(new)) # Disable GPG verification if options.no_gpg: logger.warn("GPG VERIFICATION DISABLED") config.set(APP_NAME, 'gpg', 'False') # Log config except the password # and proxy as it might have a pw as well for item, value in config.items(APP_NAME): if item != 'password' and item != 'proxy': logger.debug("%s:%s", item, value) if config.getboolean(APP_NAME, 'auto_update'): options.update = True if config.getboolean(APP_NAME, 'auto_config'): # Try to discover if we are connected to a satellite or not try_auto_configuration(config) # Set the schedule InsightsSchedule(options) # Test connection, useful for proxy debug if options.test_connection: pconn = InsightsConnection(config) pconn._test_connection() # Handle registration and grouping, this is mostly a no-op if options.register: opt_group = options.group if options.satellite_group: opt_group = get_satellite_group() hostname, opt_group = register(config, opt_group) logger.info('Successfully registered %s in group %s', hostname, opt_group) # Check for .unregistered file if os.path.isfile(constants.unregistered_file): logger.error("This machine has been unregistered") logger.error("Use --register if you would like to re-register this machine") logger.error("Exiting") sys.exit(1) # If we are not just setting the schedule, do work son if not options.schedule: collect_data_and_upload(config, options) handler.doRollover()
def collect_data_and_upload(config, options, rc=0): """ All the heavy lifting done here """ collection_start = time.clock() pconn = InsightsConnection(config) try: branch_info = pconn.branch_info() except requests.ConnectionError: branch_info = handle_branch_info_error( "Could not connect to determine branch information", options) except LookupError: branch_info = handle_branch_info_error( "Could not determine branch information", options) pc = InsightsConfig(config, pconn) archive = InsightsArchive(compressor=options.compressor) dc = DataCollector(archive) # register the exit handler here to delete the archive atexit.register(handle_exit, archive, options.keep_archive or options.no_upload) stdin_config = json.load(sys.stdin) if options.from_stdin else {} start = time.clock() collection_rules, rm_conf = pc.get_conf(options.update, stdin_config) elapsed = (time.clock() - start) logger.debug("Collection Rules Elapsed Time: %s", elapsed) start = time.clock() logger.info('Starting to collect Insights data') dc.run_commands(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("Command Collection Elapsed Time: %s", elapsed) start = time.clock() dc.copy_files(collection_rules, rm_conf) elapsed = (time.clock() - start) logger.debug("File Collection Elapsed Time: %s", elapsed) dc.write_branch_info(branch_info) obfuscate = config.getboolean(APP_NAME, "obfuscate") collection_duration = (time.clock() - collection_start) if not options.no_tar_file: tar_file = dc.done(config, rm_conf) if not options.no_upload: logger.info('Uploading Insights data,' ' this may take a few minutes') for tries in range(options.retries): upload = pconn.upload_archive(tar_file, collection_duration) if upload.status_code == 201: write_lastupload_file() logger.info("Upload completed successfully!") break elif upload.status_code == 412: pconn.handle_fail_rcs(upload) else: logger.error("Upload attempt %d of %d failed! Status Code: %s", tries + 1, options.retries, upload.status_code) if tries + 1 != options.retries: logger.info("Waiting %d seconds then retrying", constants.sleep_time) time.sleep(constants.sleep_time) else: logger.error("All attempts to upload have failed!") logger.error("Please see %s for additional information", constants.default_log_file) rc = 1 if (not obfuscate and not options.keep_archive) or options.no_upload: dc.archive.delete_tmp_dir() else: if obfuscate: logger.info('Obfuscated Insights data retained in %s', os.path.dirname(tar_file)) else: logger.info('Insights data retained in %s', tar_file) else: handle_file_output(options, tar_file) else: logger.info('See Insights data in %s', dc.archive.archive_dir) return rc
def handle_startup(): """ Handle startup options """ # ----do X and exit options---- # show version and exit if InsightsClient.options.version: print constants.version sys.exit() if (InsightsClient.options.container_mode and not InsightsClient.options.run_here and insights_client_container_is_available()): sys.exit(run_in_container()) if InsightsClient.options.validate: validate_remove_file() sys.exit() if InsightsClient.options.enable_schedule and InsightsClient.options.disable_schedule: logger.error('Conflicting options: --enable-schedule and --disable-schedule') sys.exit(1) if InsightsClient.options.enable_schedule: # enable automatic scheduling InsightsSchedule(container_mode=options.container_mode) InsightsClient.config.set(APP_NAME, 'no_schedule', False) logger.info('Automatic scheduling for Insights has been enabled.') sys.exit() if InsightsClient.options.disable_schedule: # disable automatic schedling InsightsSchedule(set_cron=False).remove_scheduling() InsightsClient.config.set(APP_NAME, 'no_schedule', True) logger.info('Automatic scheduling for Insights has been disabled.') sys.exit() # do auto_config here, for connection-related 'do X and exit' options if InsightsClient.config.getboolean(APP_NAME, 'auto_config') and not InsightsClient.options.offline: # Try to discover if we are connected to a satellite or not try_auto_configuration() if InsightsClient.options.test_connection: pconn = InsightsConnection() rc = pconn.test_connection() sys.exit(rc) if InsightsClient.options.status: reg_check = registration_check() logger.info('\n'.join(reg_check['messages'])) # exit with !status, 0 for True, 1 for False sys.exit(not reg_check['status']) if InsightsClient.options.support: support = InsightsSupport() support.collect_support_info() sys.exit() # ----config options---- # log the config # ignore password and proxy -- proxy might have pw for item, value in InsightsClient.config.items(APP_NAME): if item != 'password' and item != 'proxy': logger.debug("%s:%s", item, value) if InsightsClient.config.getboolean(APP_NAME, 'auto_update') and not InsightsClient.options.offline: # TODO: config updates option, but in GPG option, the option updates # the config. make this consistent InsightsClient.options.update = True # disable automatic scheduling if it was set in the config, and if the job exists if InsightsClient.config.getboolean(APP_NAME, 'no_schedule'): cron = InsightsSchedule(set_cron=False) if cron.already_linked(): cron.remove_scheduling() logger.debug('Automatic scheduling for Insights has been disabled.') # ----modifier options---- if InsightsClient.options.no_gpg: logger.warn("WARNING: GPG VERIFICATION DISABLED") InsightsClient.config.set(APP_NAME, 'gpg', 'False') if InsightsClient.options.just_upload: # override these for great justice InsightsClient.options.no_tar_file = False InsightsClient.options.keep_archive = True if InsightsClient.options.container_mode and InsightsClient.options.no_tar_file: logger.error('Invalid combination: --container and --no-tar-file') sys.exit(1) # can't use bofa if InsightsClient.options.from_stdin and InsightsClient.options.from_file: logger.error('Can\'t use both --from-stdin and --from-file.') sys.exit(1) # ----register options---- # put this first to avoid conflicts with register if InsightsClient.options.unregister: pconn = InsightsConnection() pconn.unregister() sys.exit() # force-reregister -- remove machine-id files and registration files before trying to register again new = False if InsightsClient.options.reregister: new = True InsightsClient.options.register = True delete_registered_file() delete_unregistered_file() delete_machine_id() logger.debug('Machine-id: %s', generate_machine_id(new)) if InsightsClient.options.register: try_register() if not InsightsClient.config.getboolean('no_schedule'): InsightsSchedule(container_mode=options.container_mode) # check registration before doing any uploads # Ignore if in offline mode if not InsightsClient.options.register and not InsightsClient.options.offline: msg, is_registered = _is_client_registered() if not is_registered: logger.error(msg) sys.exit(1)
def handle_startup(options, config): """ Handle startup options """ if options.version: print constants.version sys.exit() if options.validate: validate_remove_file() sys.exit() # Generate /etc/machine-id if it does not exist new = False if options.reregister: new = True options.register = True logger.debug("Machine-ID: " + generate_machine_id(new)) # Disable GPG verification if options.no_gpg: logger.warn("WARNING: GPG VERIFICATION DISABLED") config.set(APP_NAME, 'gpg', 'False') # Log config except the password # and proxy as it might have a pw as well for item, value in config.items(APP_NAME): if item != 'password' and item != 'proxy': logger.debug("%s:%s", item, value) if config.getboolean(APP_NAME, 'auto_update'): options.update = True if config.getboolean(APP_NAME, 'auto_config'): # Try to discover if we are connected to a satellite or not try_auto_configuration(config) # Set the schedule if not options.no_schedule and not config.getboolean( APP_NAME, 'no_schedule'): InsightsSchedule() # Test connection, useful for proxy debug if options.test_connection: pconn = InsightsConnection(config) pconn.test_connection() if options.unregister: pconn = InsightsConnection(config) pconn.unregister() sys.exit() # Handle registration, grouping, and display name if options.register: opt_group = options.group message, hostname, opt_group, display_name = register(config, options) if options.display_name is None and options.group is None: logger.info('Successfully registered %s', hostname) elif options.display_name is None: logger.info('Successfully registered %s in group %s', hostname, opt_group) else: logger.info('Successfully registered %s as %s in group %s', hostname, display_name, opt_group) logger.info(message) # Collect debug/log information if options.support: support = InsightsSupport(config) support.collect_support_info() sys.exit(0) # Just check registration status if options.status: support = InsightsSupport(config) reg_check = support.registration_check() logger.info('\n'.join(reg_check)) sys.exit(0) # Set offline mode for OSP use offline_mode = False if (options.offline and options.from_stdin) or options.no_upload: offline_mode = True # First startup, no .registered or .unregistered # Ignore if in offline mode if (not os.path.isfile(constants.registered_file) and not os.path.isfile(constants.unregistered_file) and not options.register and not offline_mode): logger.error('This machine has not yet been registered.') logger.error('Use --register to register this machine.') logger.error("Exiting") sys.exit(1) # Check for .unregistered file if (os.path.isfile(constants.unregistered_file) and not options.register and not offline_mode): logger.error("This machine has been unregistered.") logger.error("Use --register if you would like to re-register this machine.") logger.error("Exiting") sys.exit(1)
def get_connection(): global INSIGHTS_CONNECTION if INSIGHTS_CONNECTION is None: INSIGHTS_CONNECTION = InsightsConnection() return INSIGHTS_CONNECTION