def init_options_and_registry(args, command_modules): """Register command modules and determine options from commandline. These are coupled together for implementation simplicity. Conceptually they are unrelated but they share implementation details that can be encapsulated by combining them this way. Args: args: [list of command-line arguments] command_modules: See make_registry. Returns: options, registry Where: options: [Namespace] From parsed args. registry: [dict] of (<command-name>: <CommandFactory>) """ args, defaults = preprocess_args(args) parser = argparse.ArgumentParser(prog='buildtool.sh') add_standard_parser_args(parser, defaults) MetricsManager.init_argument_parser(parser, defaults) registry = make_registry(command_modules, parser, defaults) options = parser.parse_args(args) options.program = 'buildtool' # Determine the version for monitoring purposes. # Depending on the options defined, this is either the branch or bom prefix. add_monitoring_context_labels(options) return options, registry
def init_options_and_registry(args, command_modules): """Register command modules and determine options from commandline. These are coupled together for implementation simplicity. Conceptually they are unrelated but they share implementation details that can be encapsulated by combining them this way. Args: args: [list of command-line arguments] command_modules: See make_registry. Returns: options, registry Where: options: [Namespace] From parsed args. registry: [dict] of (<command-name>: <CommandFactory>) """ args, defaults = preprocess_args(args) parser = argparse.ArgumentParser(prog='buildtool.sh') init_standard_parser(parser, defaults) MetricsManager.init_argument_parser(parser, defaults) registry = make_registry(command_modules, parser, defaults) return parser.parse_args(args), registry
def __init__(self, message, classification, cause=None): super(BuildtoolError, self).__init__(message) labels = { 'cause': cause, 'classification': classification } MetricsManager.singleton().inc_counter('BuildtoolError', labels)
def scan_logs_for_install_errors(path): """Scan logfile at path and count specific errors of interest.""" content = io.open(path, 'r', encoding='utf-8').read() match = re.search( "^E:.* Version '([^']+)' for '([^']+)' was not found", content, re.MULTILINE) component = '' cause = 'Unknown' if match: version = match.group(1) component = match.group(2) cause = 'ComponentNotFound' logging.error('"%s" version "%s" does not exist.', component, version) if not match: match = re.search( '.*: No such file or directory$', content, re.MULTILINE) if match: cause = 'FileNotFound' labels = { 'component': component, 'cause': cause } MetricsManager.singleton().inc_counter('InstallSpinnakerError', labels)
def get_options(args): """Resolve all the command-line options.""" args, defaults = preprocess_args( args, default_home_path_filename='validate_bom.yml') parser = argparse.ArgumentParser(prog='validate_bom.sh') add_standard_parser_args(parser, defaults) # DEPRECATED - use output_dir instead add_parser_argument(parser, 'log_dir', defaults, './validate_bom_results', help='Path to root directory for report output.') MetricsManager.init_argument_parser(parser, defaults) validate_bom__config.init_argument_parser(parser, defaults) validate_bom__deploy.init_argument_parser(parser, defaults) validate_bom__test.init_argument_parser(parser, defaults) options = parser.parse_args(args) options.program = 'validate_bom' options.command = 'validate_bom' # metrics assumes a "command" value. options.log_dir = options.output_dir # deprecated validate_bom__config.validate_options(options) validate_bom__test.validate_options(options) if not os.path.exists(options.log_dir): os.makedirs(options.log_dir) if options.influxdb_database == 'SpinnakerBuildTool': options.influxdb_database = 'SpinnakerValidate' # Add platform/spinnaker_type to each metric we produce. # We'll use this to distinguish what was being tested. context_labels = 'platform=%s,deployment_type=%s' % ( validate_bom__deploy.determine_deployment_platform(options), options.deploy_spinnaker_type) latest_unvalidated_suffix = '-latest-unvalidated' if options.deploy_version.endswith(latest_unvalidated_suffix): bom_series = options.deploy_version[:-len(latest_unvalidated_suffix)] else: bom_series = options.deploy_version[:options.deploy_version.rfind('-')] context_labels += ',version=%s' % bom_series if options.monitoring_context_labels: context_labels += ',' + options.monitoring_context_labels options.monitoring_context_labels = context_labels return options
def main(): """The main command dispatcher.""" GitRunner.stash_and_clear_auth_env_vars() import buildtool.source_commands import buildtool.build_commands import buildtool.bom_commands import buildtool.changelog_commands import buildtool.apidocs_commands import buildtool.image_commands command_modules = [ buildtool.source_commands, buildtool.build_commands, buildtool.bom_commands, buildtool.changelog_commands, buildtool.apidocs_commands, buildtool.image_commands ] options, command_registry = init_options_and_registry( sys.argv[1:], command_modules) logging.basicConfig( format='%(levelname).1s %(asctime)s.%(msecs)03d' ' [%(threadName)s.%(process)d] %(message)s', datefmt='%H:%M:%S', level=STANDARD_LOG_LEVELS[options.log_level]) logging.debug( 'Running with options:\n %s', '\n '.join(yaml.dump(vars(options), default_flow_style=False) .split('\n'))) factory = command_registry.get(options.command) if not factory: logging.error('Unknown command "%s"', options.command) return -1 MetricsManager.startup_metrics(options) try: command = factory.make_command(options) command() finally: MetricsManager.shutdown_metrics() return 0
def wrapped_main(): options = get_options(sys.argv[1:]) logging.basicConfig(format='%(levelname).1s %(asctime)s.%(msecs)03d' ' [%(threadName)s.%(process)d] %(message)s', datefmt='%H:%M:%S', level=STANDARD_LOG_LEVELS[options.log_level]) logging.debug( 'Running with options:\n %s', '\n '.join( yaml.dump(vars(options), default_flow_style=False).split('\n'))) metrics = MetricsManager.startup_metrics(options) try: return main(options, metrics) finally: MetricsManager.shutdown_metrics()
def scan_logs_for_install_errors(path): """Scan logfile at path and count specific errors of interest.""" content = open(path, 'r').read() match = re.search("^E:.* Version '([^']+)' for '([^']+)' was not found", content, re.MULTILINE) component = '' cause = 'Unknown' if match: version = match.group(1) component = match.group(2) cause = 'ComponentNotFound' logging.error('"%s" version "%s" does not exist.', component, version) if not match: match = re.search('.*: No such file or directory$', content, re.MULTILINE) if match: cause = 'FileNotFound' labels = {'component': component, 'cause': cause} MetricsManager.singleton().inc_counter('InstallSpinnakerError', labels)
def scan_logs_for_install_errors(path): """Scan logfile at path and count specific errors of interest.""" content = io.open(path, "r", encoding="utf-8").read() match = re.search("^E:.* Version '([^']+)' for '([^']+)' was not found", content, re.MULTILINE) component = "" cause = "Unknown" if match: version = match.group(1) component = match.group(2) cause = "ComponentNotFound" logging.error('"%s" version "%s" does not exist.', component, version) if not match: match = re.search(".*: No such file or directory$", content, re.MULTILINE) if match: cause = "FileNotFound" labels = {"component": component, "cause": cause} MetricsManager.singleton().inc_counter("InstallSpinnakerError", labels)
def main(): """The main command dispatcher.""" start_time = time.time() from importlib import import_module command_modules = [ import_module(name + '_commands') for name in [ 'apidocs', 'bom', 'changelog', 'container', 'debian', 'halyard', 'image', 'rpm', 'source', 'spinnaker', 'inspection', 'spin', ]] GitRunner.stash_and_clear_auth_env_vars() options, command_registry = init_options_and_registry( sys.argv[1:], command_modules) logging.basicConfig( format='%(levelname).1s %(asctime)s.%(msecs)03d' ' [%(threadName)s.%(process)d] %(message)s', datefmt='%H:%M:%S', level=STANDARD_LOG_LEVELS[options.log_level]) logging.debug( 'Running with options:\n %s', '\n '.join(yaml.safe_dump(vars(options), default_flow_style=False) .split('\n'))) factory = command_registry.get(options.command) if not factory: logging.error('Unknown command "%s"', options.command) return -1 MetricsManager.startup_metrics(options) labels = {'command': options.command} success = False try: command = factory.make_command(options) command() success = True finally: labels['success'] = success MetricsManager.singleton().observe_timer( 'BuildTool_Outcome', labels, time.time() - start_time) MetricsManager.shutdown_metrics() return 0
def main(): """The main command dispatcher.""" start_time = time.time() from importlib import import_module command_modules = [ import_module(name + "_commands") for name in [ "apidocs", "bom", "changelog", "halyard", "image", "source", "spinnaker", "inspection", ] ] GitRunner.stash_and_clear_auth_env_vars() options, command_registry = init_options_and_registry( sys.argv[1:], command_modules) logging.basicConfig( format="%(levelname).1s %(asctime)s.%(msecs)03d" " [%(threadName)s.%(process)d] %(message)s", datefmt="%H:%M:%S", level=STANDARD_LOG_LEVELS[options.log_level], ) logging.debug( "Running with options:\n %s", "\n ".join( yaml.safe_dump(vars(options), default_flow_style=False).split("\n")), ) factory = command_registry.get(options.command) if not factory: logging.error('Unknown command "%s"', options.command) return -1 MetricsManager.startup_metrics(options) labels = {"command": options.command} success = False try: command = factory.make_command(options) command() success = True finally: labels["success"] = success MetricsManager.singleton().observe_timer("BuildTool_Outcome", labels, time.time() - start_time) MetricsManager.shutdown_metrics() return 0
def __init__(self, factory, options): self.__factory = factory self.__options = options self.__metrics = MetricsManager.singleton()
def __init__(self, message, classification, cause=None): super(BuildtoolError, self).__init__(message) labels = {"cause": cause, "classification": classification} MetricsManager.singleton().inc_counter("BuildtoolError", labels)