def test_failures_cancel_flush(self, caplog, num_failures,
expected_log_count, expected_cancel):
"""Test that too many failures will cancel further processing on flush.
2 messages should not cancel on flush
3 or more should cancel on flush
The number of received messages will be based on how many have
been processed before the flush was initiated.
"""
responses.add(responses.POST, "http://localhost", status=404)
for _ in range(num_failures):
report_start_event("name", "description")
flush_events()
# Force a context switch. Without this, it's possible that the
# expected log message hasn't made it to the log file yet
time.sleep(0.01)
# If we've pushed a bunch of messages, any number could have been
# processed before we get to the flush.
assert (expected_log_count <= caplog.text.count("Failed posting event")
<= num_failures)
cancelled_message = (
"Multiple consecutive failures in WebHookHandler. "
"Cancelling all queued events")
if expected_cancel:
assert cancelled_message in caplog.text
else:
assert cancelled_message not in caplog.text
def test_webhook_handler(self, caplog):
"""Test the happy path."""
responses.add(responses.POST, "http://localhost", status=200)
report_start_event("name", "description")
flush_events()
assert 1 == caplog.text.count(
"Read from http://localhost (200, 0b) after 1 attempts")
def handle_args(name, args):
# Note that if an exception happens between now and when logging is
# setup, we'll only see it in the journal
hotplug_reporter = events.ReportEventStack(name,
__doc__,
reporting_enabled=True)
hotplug_init = Init(ds_deps=[], reporter=hotplug_reporter)
hotplug_init.read_cfg()
log.setupLogging(hotplug_init.cfg)
if "reporting" in hotplug_init.cfg:
reporting.update_configuration(hotplug_init.cfg.get("reporting"))
# Logging isn't going to be setup until now
LOG.debug(
"%s called with the following arguments: {"
"hotplug_action: %s, subsystem: %s, udevaction: %s, devpath: %s}",
name,
args.hotplug_action,
args.subsystem,
args.udevaction if "udevaction" in args else None,
args.devpath if "devpath" in args else None,
)
with hotplug_reporter:
try:
if args.hotplug_action == "query":
try:
datasource = initialize_datasource(hotplug_init,
args.subsystem)
except DataSourceNotFoundException:
print("Unable to determine hotplug state. No datasource "
"detected")
sys.exit(1)
print("enabled" if datasource else "disabled")
else:
handle_hotplug(
hotplug_init=hotplug_init,
devpath=args.devpath,
subsystem=args.subsystem,
udevaction=args.udevaction,
)
except Exception:
LOG.exception("Received fatal exception handling hotplug!")
raise
LOG.debug("Exiting hotplug handler")
reporting.flush_events()
def handle_args(name, args):
# Note that if an exception happens between now and when logging is
# setup, we'll only see it in the journal
hotplug_reporter = events.ReportEventStack(name,
__doc__,
reporting_enabled=True)
hotplug_init = Init(ds_deps=[], reporter=hotplug_reporter)
hotplug_init.read_cfg()
log.setupLogging(hotplug_init.cfg)
if 'reporting' in hotplug_init.cfg:
reporting.update_configuration(hotplug_init.cfg.get('reporting'))
# Logging isn't going to be setup until now
LOG.debug(
'%s called with the following arguments: {udevaction: %s, '
'subsystem: %s, devpath: %s}', name, args.udevaction, args.subsystem,
args.devpath)
LOG.debug(
'%s called with the following arguments:\n'
'udevaction: %s\n'
'subsystem: %s\n'
'devpath: %s', name, args.udevaction, args.subsystem, args.devpath)
with hotplug_reporter:
try:
handle_hotplug(
hotplug_init=hotplug_init,
devpath=args.devpath,
subsystem=args.subsystem,
udevaction=args.udevaction,
)
except Exception:
LOG.exception('Received fatal exception handling hotplug!')
raise
LOG.debug('Exiting hotplug handler')
reporting.flush_events()
def main(sysv_args=None):
if not sysv_args:
sysv_args = sys.argv
parser = argparse.ArgumentParser(prog=sysv_args.pop(0))
# Top level args
parser.add_argument(
"--version",
"-v",
action="version",
version="%(prog)s " + (version.version_string()),
help="Show program's version number and exit.",
)
parser.add_argument(
"--file",
"-f",
action="append",
dest="files",
help="Use additional yaml configuration files.",
type=argparse.FileType("rb"),
)
parser.add_argument(
"--debug",
"-d",
action="store_true",
help="Show additional pre-action logging (default: %(default)s).",
default=False,
)
parser.add_argument(
"--force",
action="store_true",
help=(
"Force running even if no datasource is"
" found (use at your own risk)."
),
dest="force",
default=False,
)
parser.set_defaults(reporter=None)
subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand")
subparsers.required = True
# Each action and its sub-options (if any)
parser_init = subparsers.add_parser(
"init", help="Initialize cloud-init and perform initial modules."
)
parser_init.add_argument(
"--local",
"-l",
action="store_true",
help="Start in local mode (default: %(default)s).",
default=False,
)
# This is used so that we can know which action is selected +
# the functor to use to run this subcommand
parser_init.set_defaults(action=("init", main_init))
# These settings are used for the 'config' and 'final' stages
parser_mod = subparsers.add_parser(
"modules", help="Activate modules using a given configuration key."
)
parser_mod.add_argument(
"--mode",
"-m",
action="store",
help="Module configuration name to use (default: %(default)s).",
default="config",
choices=("init", "config", "final"),
)
parser_mod.set_defaults(action=("modules", main_modules))
# This subcommand allows you to run a single module
parser_single = subparsers.add_parser(
"single", help="Run a single module."
)
parser_single.add_argument(
"--name",
"-n",
action="store",
help="module name to run",
required=True,
)
parser_single.add_argument(
"--frequency",
action="store",
help="Set module frequency.",
required=False,
choices=list(FREQ_SHORT_NAMES.keys()),
)
parser_single.add_argument(
"--report",
action="store_true",
help="Enable reporting.",
required=False,
)
parser_single.add_argument(
"module_args",
nargs="*",
metavar="argument",
help="Any additional arguments to pass to this module.",
)
parser_single.set_defaults(action=("single", main_single))
parser_query = subparsers.add_parser(
"query",
help="Query standardized instance metadata from the command line.",
)
parser_dhclient = subparsers.add_parser(
dhclient_hook.NAME, help=dhclient_hook.__doc__
)
dhclient_hook.get_parser(parser_dhclient)
parser_features = subparsers.add_parser(
"features", help="List defined features."
)
parser_features.set_defaults(action=("features", main_features))
parser_analyze = subparsers.add_parser(
"analyze", help="Devel tool: Analyze cloud-init logs and data."
)
parser_devel = subparsers.add_parser(
"devel", help="Run development tools."
)
parser_collect_logs = subparsers.add_parser(
"collect-logs", help="Collect and tar all cloud-init debug info."
)
parser_clean = subparsers.add_parser(
"clean", help="Remove logs and artifacts so cloud-init can re-run."
)
parser_status = subparsers.add_parser(
"status", help="Report cloud-init status or wait on completion."
)
parser_schema = subparsers.add_parser(
"schema", help="Validate cloud-config files using jsonschema."
)
if sysv_args:
# Only load subparsers if subcommand is specified to avoid load cost
subcommand = sysv_args[0]
if subcommand == "analyze":
from cloudinit.analyze.__main__ import get_parser as analyze_parser
# Construct analyze subcommand parser
analyze_parser(parser_analyze)
elif subcommand == "devel":
from cloudinit.cmd.devel.parser import get_parser as devel_parser
# Construct devel subcommand parser
devel_parser(parser_devel)
elif subcommand == "collect-logs":
from cloudinit.cmd.devel.logs import (
get_parser as logs_parser,
handle_collect_logs_args,
)
logs_parser(parser_collect_logs)
parser_collect_logs.set_defaults(
action=("collect-logs", handle_collect_logs_args)
)
elif subcommand == "clean":
from cloudinit.cmd.clean import (
get_parser as clean_parser,
handle_clean_args,
)
clean_parser(parser_clean)
parser_clean.set_defaults(action=("clean", handle_clean_args))
elif subcommand == "query":
from cloudinit.cmd.query import (
get_parser as query_parser,
handle_args as handle_query_args,
)
query_parser(parser_query)
parser_query.set_defaults(action=("render", handle_query_args))
elif subcommand == "schema":
from cloudinit.config.schema import (
get_parser as schema_parser,
handle_schema_args,
)
schema_parser(parser_schema)
parser_schema.set_defaults(action=("schema", handle_schema_args))
elif subcommand == "status":
from cloudinit.cmd.status import (
get_parser as status_parser,
handle_status_args,
)
status_parser(parser_status)
parser_status.set_defaults(action=("status", handle_status_args))
args = parser.parse_args(args=sysv_args)
# Subparsers.required = True and each subparser sets action=(name, functor)
(name, functor) = args.action
# Setup basic logging to start (until reinitialized)
# iff in debug mode.
if args.debug:
logging.setupBasicLogging()
# Setup signal handlers before running
signal_handler.attach_handlers()
if name in ("modules", "init"):
functor = status_wrapper
rname = None
report_on = True
if name == "init":
if args.local:
rname, rdesc = ("init-local", "searching for local datasources")
else:
rname, rdesc = (
"init-network",
"searching for network datasources",
)
elif name == "modules":
rname, rdesc = (
"modules-%s" % args.mode,
"running modules for %s" % args.mode,
)
elif name == "single":
rname, rdesc = (
"single/%s" % args.name,
"running single module %s" % args.name,
)
report_on = args.report
else:
rname = name
rdesc = "running 'cloud-init %s'" % name
report_on = False
args.reporter = events.ReportEventStack(
rname, rdesc, reporting_enabled=report_on
)
with args.reporter:
retval = util.log_time(
logfunc=LOG.debug,
msg="cloud-init mode '%s'" % name,
get_uptime=True,
func=functor,
args=(name, args),
)
reporting.flush_events()
return retval
def main(sysv_args=None):
if not sysv_args:
sysv_args = sys.argv
parser = argparse.ArgumentParser(prog=sysv_args[0])
sysv_args = sysv_args[1:]
# Top level args
parser.add_argument('--version',
'-v',
action='version',
version='%(prog)s ' + (version.version_string()))
parser.add_argument('--file',
'-f',
action='append',
dest='files',
help=('additional yaml configuration'
' files to use'),
type=argparse.FileType('rb'))
parser.add_argument('--debug',
'-d',
action='store_true',
help=('show additional pre-action'
' logging (default: %(default)s)'),
default=False)
parser.add_argument('--force',
action='store_true',
help=('force running even if no datasource is'
' found (use at your own risk)'),
dest='force',
default=False)
parser.set_defaults(reporter=None)
subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
subparsers.required = True
# Each action and its sub-options (if any)
parser_init = subparsers.add_parser('init',
help=('initializes cloud-init and'
' performs initial modules'))
parser_init.add_argument("--local",
'-l',
action='store_true',
help="start in local mode (default: %(default)s)",
default=False)
# This is used so that we can know which action is selected +
# the functor to use to run this subcommand
parser_init.set_defaults(action=('init', main_init))
# These settings are used for the 'config' and 'final' stages
parser_mod = subparsers.add_parser('modules',
help=('activates modules using '
'a given configuration key'))
parser_mod.add_argument("--mode",
'-m',
action='store',
help=("module configuration name "
"to use (default: %(default)s)"),
default='config',
choices=('init', 'config', 'final'))
parser_mod.set_defaults(action=('modules', main_modules))
# This subcommand allows you to run a single module
parser_single = subparsers.add_parser('single',
help=('run a single module '))
parser_single.add_argument("--name",
'-n',
action="store",
help="module name to run",
required=True)
parser_single.add_argument("--frequency",
action="store",
help=("frequency of the module"),
required=False,
choices=list(FREQ_SHORT_NAMES.keys()))
parser_single.add_argument("--report",
action="store_true",
help="enable reporting",
required=False)
parser_single.add_argument("module_args",
nargs="*",
metavar='argument',
help=('any additional arguments to'
' pass to this module'))
parser_single.set_defaults(action=('single', main_single))
parser_query = subparsers.add_parser(
'query',
help='Query standardized instance metadata from the command line.')
parser_dhclient = subparsers.add_parser(dhclient_hook.NAME,
help=dhclient_hook.__doc__)
dhclient_hook.get_parser(parser_dhclient)
parser_features = subparsers.add_parser('features',
help=('list defined features'))
parser_features.set_defaults(action=('features', main_features))
parser_analyze = subparsers.add_parser(
'analyze', help='Devel tool: Analyze cloud-init logs and data')
parser_devel = subparsers.add_parser('devel', help='Run development tools')
parser_collect_logs = subparsers.add_parser(
'collect-logs', help='Collect and tar all cloud-init debug info')
parser_clean = subparsers.add_parser(
'clean', help='Remove logs and artifacts so cloud-init can re-run.')
parser_status = subparsers.add_parser(
'status', help='Report cloud-init status or wait on completion.')
if sysv_args:
# Only load subparsers if subcommand is specified to avoid load cost
if sysv_args[0] == 'analyze':
from cloudinit.analyze.__main__ import get_parser as analyze_parser
# Construct analyze subcommand parser
analyze_parser(parser_analyze)
elif sysv_args[0] == 'devel':
from cloudinit.cmd.devel.parser import get_parser as devel_parser
# Construct devel subcommand parser
devel_parser(parser_devel)
elif sysv_args[0] == 'collect-logs':
from cloudinit.cmd.devel.logs import (get_parser as logs_parser,
handle_collect_logs_args)
logs_parser(parser_collect_logs)
parser_collect_logs.set_defaults(action=('collect-logs',
handle_collect_logs_args))
elif sysv_args[0] == 'clean':
from cloudinit.cmd.clean import (get_parser as clean_parser,
handle_clean_args)
clean_parser(parser_clean)
parser_clean.set_defaults(action=('clean', handle_clean_args))
elif sysv_args[0] == 'query':
from cloudinit.cmd.query import (get_parser as query_parser,
handle_args as handle_query_args)
query_parser(parser_query)
parser_query.set_defaults(action=('render', handle_query_args))
elif sysv_args[0] == 'status':
from cloudinit.cmd.status import (get_parser as status_parser,
handle_status_args)
status_parser(parser_status)
parser_status.set_defaults(action=('status', handle_status_args))
args = parser.parse_args(args=sysv_args)
# Subparsers.required = True and each subparser sets action=(name, functor)
(name, functor) = args.action
# Setup basic logging to start (until reinitialized)
# iff in debug mode.
if args.debug:
logging.setupBasicLogging()
# Setup signal handlers before running
signal_handler.attach_handlers()
if name in ("modules", "init"):
functor = status_wrapper
rname = None
report_on = True
if name == "init":
if args.local:
rname, rdesc = ("init-local", "searching for local datasources")
else:
rname, rdesc = ("init-network",
"searching for network datasources")
elif name == "modules":
rname, rdesc = ("modules-%s" % args.mode,
"running modules for %s" % args.mode)
elif name == "single":
rname, rdesc = ("single/%s" % args.name,
"running single module %s" % args.name)
report_on = args.report
else:
rname = name
rdesc = "running 'cloud-init %s'" % name
report_on = False
args.reporter = events.ReportEventStack(rname,
rdesc,
reporting_enabled=report_on)
with args.reporter:
retval = util.log_time(logfunc=LOG.debug,
msg="cloud-init mode '%s'" % name,
get_uptime=True,
func=functor,
args=(name, args))
reporting.flush_events()
return retval
def main(sysv_args=None):
if not sysv_args:
sysv_args = sys.argv
parser = argparse.ArgumentParser(prog=sysv_args[0])
sysv_args = sysv_args[1:]
# Top level args
parser.add_argument('--version', '-v', action='version',
version='%(prog)s ' + (version.version_string()))
parser.add_argument('--file', '-f', action='append',
dest='files',
help=('additional yaml configuration'
' files to use'),
type=argparse.FileType('rb'))
parser.add_argument('--debug', '-d', action='store_true',
help=('show additional pre-action'
' logging (default: %(default)s)'),
default=False)
parser.add_argument('--force', action='store_true',
help=('force running even if no datasource is'
' found (use at your own risk)'),
dest='force',
default=False)
parser.set_defaults(reporter=None)
subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
subparsers.required = True
# Each action and its sub-options (if any)
parser_init = subparsers.add_parser('init',
help=('initializes cloud-init and'
' performs initial modules'))
parser_init.add_argument("--local", '-l', action='store_true',
help="start in local mode (default: %(default)s)",
default=False)
# This is used so that we can know which action is selected +
# the functor to use to run this subcommand
parser_init.set_defaults(action=('init', main_init))
# These settings are used for the 'config' and 'final' stages
parser_mod = subparsers.add_parser('modules',
help=('activates modules using '
'a given configuration key'))
parser_mod.add_argument("--mode", '-m', action='store',
help=("module configuration name "
"to use (default: %(default)s)"),
default='config',
choices=('init', 'config', 'final'))
parser_mod.set_defaults(action=('modules', main_modules))
# This subcommand allows you to run a single module
parser_single = subparsers.add_parser('single',
help=('run a single module '))
parser_single.add_argument("--name", '-n', action="store",
help="module name to run",
required=True)
parser_single.add_argument("--frequency", action="store",
help=("frequency of the module"),
required=False,
choices=list(FREQ_SHORT_NAMES.keys()))
parser_single.add_argument("--report", action="store_true",
help="enable reporting",
required=False)
parser_single.add_argument("module_args", nargs="*",
metavar='argument',
help=('any additional arguments to'
' pass to this module'))
parser_single.set_defaults(action=('single', main_single))
parser_query = subparsers.add_parser(
'query',
help='Query standardized instance metadata from the command line.')
parser_dhclient = subparsers.add_parser(
dhclient_hook.NAME, help=dhclient_hook.__doc__)
dhclient_hook.get_parser(parser_dhclient)
parser_features = subparsers.add_parser('features',
help=('list defined features'))
parser_features.set_defaults(action=('features', main_features))
parser_analyze = subparsers.add_parser(
'analyze', help='Devel tool: Analyze cloud-init logs and data')
parser_devel = subparsers.add_parser(
'devel', help='Run development tools')
parser_collect_logs = subparsers.add_parser(
'collect-logs', help='Collect and tar all cloud-init debug info')
parser_clean = subparsers.add_parser(
'clean', help='Remove logs and artifacts so cloud-init can re-run.')
parser_status = subparsers.add_parser(
'status', help='Report cloud-init status or wait on completion.')
if sysv_args:
# Only load subparsers if subcommand is specified to avoid load cost
if sysv_args[0] == 'analyze':
from cloudinit.analyze.__main__ import get_parser as analyze_parser
# Construct analyze subcommand parser
analyze_parser(parser_analyze)
elif sysv_args[0] == 'devel':
from cloudinit.cmd.devel.parser import get_parser as devel_parser
# Construct devel subcommand parser
devel_parser(parser_devel)
elif sysv_args[0] == 'collect-logs':
from cloudinit.cmd.devel.logs import (
get_parser as logs_parser, handle_collect_logs_args)
logs_parser(parser_collect_logs)
parser_collect_logs.set_defaults(
action=('collect-logs', handle_collect_logs_args))
elif sysv_args[0] == 'clean':
from cloudinit.cmd.clean import (
get_parser as clean_parser, handle_clean_args)
clean_parser(parser_clean)
parser_clean.set_defaults(
action=('clean', handle_clean_args))
elif sysv_args[0] == 'query':
from cloudinit.cmd.query import (
get_parser as query_parser, handle_args as handle_query_args)
query_parser(parser_query)
parser_query.set_defaults(
action=('render', handle_query_args))
elif sysv_args[0] == 'status':
from cloudinit.cmd.status import (
get_parser as status_parser, handle_status_args)
status_parser(parser_status)
parser_status.set_defaults(
action=('status', handle_status_args))
args = parser.parse_args(args=sysv_args)
# Subparsers.required = True and each subparser sets action=(name, functor)
(name, functor) = args.action
# Setup basic logging to start (until reinitialized)
# iff in debug mode.
if args.debug:
logging.setupBasicLogging()
# Setup signal handlers before running
signal_handler.attach_handlers()
if name in ("modules", "init"):
functor = status_wrapper
rname = None
report_on = True
if name == "init":
if args.local:
rname, rdesc = ("init-local", "searching for local datasources")
else:
rname, rdesc = ("init-network",
"searching for network datasources")
elif name == "modules":
rname, rdesc = ("modules-%s" % args.mode,
"running modules for %s" % args.mode)
elif name == "single":
rname, rdesc = ("single/%s" % args.name,
"running single module %s" % args.name)
report_on = args.report
else:
rname = name
rdesc = "running 'cloud-init %s'" % name
report_on = False
args.reporter = events.ReportEventStack(
rname, rdesc, reporting_enabled=report_on)
with args.reporter:
retval = util.log_time(
logfunc=LOG.debug, msg="cloud-init mode '%s'" % name,
get_uptime=True, func=functor, args=(name, args))
reporting.flush_events()
return retval
def test_404(self, caplog):
"""Test failure"""
responses.add(responses.POST, "http://localhost", status=404)
report_start_event("name", "description")
flush_events()
assert 1 == caplog.text.count("Failed posting event")
