def get_iam(args=None, logger=None, stats=None):
"""
Return a usable IAM object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones.
(If you omit the add_iam_cli_arguments() call during other cli setup,
the Boto object will still work, but its cli options won't show up in
--help output)
(This also handles instantiating a Boto3 object on its own.)
"""
if not args:
parser = get_parser(description=NAME)
add_iam_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
boto = Boto3(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
region=args.boto_region,
logger=logger,
stats=stats,
)
return IAM(
boto=boto,
logger=logger,
stats=stats,
)
def get_s3(args=None, logger=None, stats=None):
"""
Return a usable S3 object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones for the SQS setup.
(If you omit the add_s3_cli_arguments() call during other cli setup,
the Boto object will still work, but its cli options won't show up in
--help output)
(This also handles instantiating a Boto object on its own.)
"""
if not args:
parser = get_parser()
add_s3_cli_arguments(parser)
# Parse only the known arguments added by add_boto_cli_arguments().
# We only need those arguments to create Boto object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return S3(
boto=get_boto(args, logger, stats),
logger=logger,
stats=stats,
)
def get_cloud_formation(args=None, logger=None, stats=None):
"""
Return a usable CloudFormation object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones for the CloudFormation setup.
(If you omit the add_cloud_formation_cli_arguments() call during other cli setup,
the CloudFormation object will still work, but its cli options won't show up in
--help output)
(This also handles instantiating a Boto3 object on its own.)
"""
if not args:
parser = get_parser()
add_cloud_formation_cli_arguments(parser)
# Parse only the known arguments added by add_cloud_formation_cli_arguments().
# We only need those arguments to create CloudFormation object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
boto3 = Boto3(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
region=args.boto_region,
logger=logger,
stats=stats,
)
boto = Boto(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
# This boto is for S3 upload and is using a constant region,
# matching the TEMP_S3_BUCKET
region=getattr(args, 'bucket_region', CloudFormation.DEFAULT_S3_REGION),
logger=logger,
stats=stats,
)
s3 = S3(
boto=boto,
logger=logger,
stats=stats,
)
return CloudFormation(
boto=boto3,
s3=s3,
bucket_name=getattr(args, 'bucket_name', CloudFormation.DEFAULT_S3_BUCKET),
logger=logger,
stats=stats,
)
def get_boto3(args=None, logger=None, stats=None):
"""
Return a usable Boto3 object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones for the boto setup.
(If you omit the add_boto_cli_arguments() call during other cli setup,
the Boto object will still work, but its cli options won't show up in
--help output)
"""
if not args:
parser = get_parser()
add_boto_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return Boto3(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
region=args.boto_region,
logger=logger,
stats=stats,
)
def __init__(self, logger=None, stats=None, parser=None):
self.master = None
self.slaves = []
self.name = "krux-redis"
self.logger = logger or get_logger(self.name)
self.stats = stats or get_stats(prefix=self.name)
self.parser = parser or get_parser(description=self.name)
### in case we got some of the information via the CLI
self.args = self.parser.parse_args()
def _get_parser(self, args=[]):
"""
Returns a mock parser with the given arguments set
:param args: :py:class:`list` List of str CLI arguments (i.e. ['--log-level', 'debug', '--log-file', 'foo.log'])
"""
# Get the argparse namespace object with the given args
parser = cli.get_parser()
namespace = parser.parse_args(args)
# Return a mock ArgumentParser object as a parser
# It has a function called 'parse_args' with the return value is the namespace variable defined above
return MagicMock(
spec=ArgumentParser,
parse_args=MagicMock(return_value=namespace)
)
def get_cloud_health(args=None, logger=None, stats=None):
if not args:
parser = get_parser(description=NAME)
add_cloud_health_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return CloudHealth(
api_key=args.api_key,
logger=logger,
stats=stats,
)
def get_cloud_health(args=None, logger=None, stats=None):
if not args:
parser = get_parser(description=NAME)
add_cloud_health_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return CloudHealth(
api_key=args.api_key,
logger=logger,
stats=stats,
)
def __init__(self, logger = None, stats = None, parser = None, application = None):
"""
Either pass in a krux.cli.Application, or separate stats/logger/parser instances.
If you pass in none of those, we'll create new instances here.
If you pass the Application class, exit hooks can be run if you ask the scheduler
to exit after its job completes. This is recommended.
"""
### to keep line length short below
_a = application
self.application = application
self.name = getattr(_a, 'name', 'krux-scheduler')
self.logger = logger or getattr(_a, 'logger', get_logger(self.name))
self.stats = stats or getattr(_a, 'stats', get_stats( prefix = self.name))
self.parser = parser or getattr(_a, 'parser', get_parser(description = self.name))
### we keep track of the last job that was run, so we can use that
### as part of the exit status in the App.
self.last_job_event = None
### in case we got some of the information via the CLI
self.args = self.parser.parse_args()
### Own process information - always record this when we have
### a chance to do so.
self.proc = psutil.Process(os.getpid())
### the AP scheduler object
_standalone = True if not self.args.scheduler_daemonize else False
self.___apscheduler = apscheduler.scheduler.Scheduler(
standalone = _standalone,
daemonic = not _standalone,
)
### catch any events that come up
self.___setup_event_listener(
terminate_on_finish = self.args.scheduler_exit_after_job,
)
def __get_arguments(args=None, logger=None, stats=None):
"""
A helper method that generates a dictionary of arguments needed to instantiate a BaseBoto object.
The purpose of this method is to abstract out the code to handle optional CLI arguments
and not duplicate the None handling code.
:param args: Namespace of arguments parsed by argparse
:type args: argparse.Namespace
:param logger: Logger, recommended to be obtained using krux.cli.Application
:type logger: logging.Logger
:param stats: Stats, recommended to be obtained using krux.cli.Application
:type stats: kruxstatsd.StatsClient
:return: A dictionary of arguments needed for BaseBoto.__init__()
:rtype: dict
"""
if not args:
parser = get_parser()
add_boto_cli_arguments(parser)
# Parse only the known arguments added by add_boto_cli_arguments().
# We only need those arguments to create Boto object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return {
'log_level': getattr(args, 'boto_log_level', DEFAULT['log_level']()),
'access_key': getattr(args, 'boto_access_key',
DEFAULT['access_key']()),
'secret_key': getattr(args, 'boto_secret_key',
DEFAULT['secret_key']()),
'region': getattr(args, 'boto_region', DEFAULT['region']()),
'logger': logger,
'stats': stats,
}
def __get_arguments(args=None, logger=None, stats=None):
"""
A helper method that generates a dictionary of arguments needed to instantiate a BaseBoto object.
The purpose of this method is to abstract out the code to handle optional CLI arguments
and not duplicate the None handling code.
:param args: Namespace of arguments parsed by argparse
:type args: argparse.Namespace
:param logger: Logger, recommended to be obtained using krux.cli.Application
:type logger: logging.Logger
:param stats: Stats, recommended to be obtained using krux.cli.Application
:type stats: kruxstatsd.StatsClient
:return: A dictionary of arguments needed for BaseBoto.__init__()
:rtype: dict
"""
if not args:
parser = get_parser()
add_boto_cli_arguments(parser)
# Parse only the known arguments added by add_boto_cli_arguments().
# We only need those arguments to create Boto object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return {
'log_level': getattr(args, 'boto_log_level', DEFAULT['log_level']()),
'access_key': getattr(args, 'boto_access_key', DEFAULT['access_key']()),
'secret_key': getattr(args, 'boto_secret_key', DEFAULT['secret_key']()),
'region': getattr(args, 'boto_region', DEFAULT['region']()),
'logger': logger,
'stats': stats,
}
def get_kafka_manager_api(args=None, logger=None, stats=None):
"""
Return a usable Kafka Manager object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If they are not inputted,
we will provide usable ones.
"""
if not args:
parser = get_parser(description=NAME)
add_kafka_manager_api_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return KafkaManager(
hostname=args.hostname,
use_ssl=args.use_ssl,
logger=logger,
stats=stats,
)
def test_get_parser():
"""
Test getting a parser from krux.cli
"""
parser = cli.get_parser()
assert_true(parser)
def __init__(
self,
log_level=None,
access_key=None,
secret_key=None,
region=None,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
if log_level is None:
log_level = DEFAULT['log_level']()
if access_key is None:
access_key = DEFAULT['access_key']()
if secret_key is None:
secret_key = DEFAULT['secret_key']()
if region is None:
region = DEFAULT['region']()
if REGION not in os.environ:
self._logger.debug(
"There is not a default region set in your environment variables. Defaulted to '%s'", region
)
# GOTCHA: Due to backward incompatible version change in v1.0.0, the users of krux_boto may
# pass wrong credential. Make sure the passed credential via CLI is the same as one passed into this instance.
parser = get_parser()
add_boto_cli_arguments(parser)
# GOTCHA: We only care about the credential arguments and nothing else.
# Don't validate the arguments or parse other things. Let krux.cli do that.
args = parser.parse_known_args()
_access_key = getattr(args, 'boto_access_key', None)
_secret_key = getattr(args, 'boto_secret_key', None)
if _access_key is not None and _access_key != access_key:
self._logger.warn(
'You set %s as boto-access-key in CLI, but passed %s to the library. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.',
BaseBoto._hide_value(_access_key), BaseBoto._hide_value(access_key),
)
if _secret_key is not None and _secret_key != secret_key:
self._logger.warn(
'You set %s as boto-secret-key in CLI, but passed %s to the library. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.',
BaseBoto._hide_value(_secret_key), BaseBoto._hide_value(secret_key),
)
# Infer the loglevel, but set it as a property so the subclasses can
# use it to set the loglevels on the loghandlers for their implementation
self._boto_log_level = LEVELS[log_level]
# this has to be 'public', so callers can use it. It's unfortunately
# near impossible to transparently wrap this, because the boto.config
# is initialized before we get here, and all the classes do a look up
# at compile time. So overriding doesn't help.
# Wrapping doesn't work cleanly, because we 1) would have to wrap
# everything, including future features we can't know about yet, as
# well as 2) poke into the guts of the implementation classes to figure
# out connection strings etc. It's quite cumbersome.
# So for now, we just store the region that was asked for, and let the
# caller use it. See the sample app for a howto.
self.cli_region = region
# if these are set, make sure we set the environment again
# as well; that way the underlying boto calls will just DTRT
# without the need to wrap all the functions.
credential_map = {
ACCESS_KEY: access_key,
SECRET_KEY: secret_key,
}
for env_var, val in iteritems(credential_map):
if val is None or len(val) < 1:
self._logger.debug('Passed boto credentials is empty. Falling back to environment variable %s', env_var)
else:
# this way we can tell what credentials are being used,
# without dumping the whole secret into the logs
self._logger.debug('Setting boto credential %s to %s', env_var, BaseBoto._hide_value(val))
os.environ[env_var] = val
# If at this point the environment variable is NOT set,
# you didn't set it, and we didn't set it. At which point
# boto will go off spelunking for .boto files or other
# settings. Best be clear about this. Using 'if not' because
# if you set it like this:
# $ FOO= ./myprog.py
# It'll return an empty string, and we'd not catch it.
if not os.environ.get(env_var, None):
self._logger.debug(
'Boto environment credential %s NOT explicitly set ' +
'-- boto will look for a .boto file somewhere', env_var
)
def __init__(
self,
log_level=None,
access_key=None,
secret_key=None,
region=None,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
if log_level is None:
log_level = DEFAULT['log_level']()
if access_key is None:
access_key = DEFAULT['access_key']()
if secret_key is None:
secret_key = DEFAULT['secret_key']()
if region is None:
region = DEFAULT['region']()
if REGION not in os.environ:
self._logger.debug(
"There is not a default region set in your environment variables. Defaulted to '%s'",
region)
# GOTCHA: Due to backward incompatible version change in v1.0.0, the users of krux_boto may
# pass wrong credential. Make sure the passed credential via CLI is the same as one passed into this instance.
parser = get_parser()
add_boto_cli_arguments(parser)
# GOTCHA: We only care about the credential arguments and nothing else.
# Don't validate the arguments or parse other things. Let krux.cli do that.
args = parser.parse_known_args()
_access_key = getattr(args, 'boto_access_key', None)
_secret_key = getattr(args, 'boto_secret_key', None)
if _access_key is not None and _access_key != access_key:
self._logger.warn(
'You set a different boto-access-key in CLI. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.')
if _secret_key is not None and _secret_key != secret_key:
self._logger.warn(
'You set a different boto-secret-key in CLI. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.')
# Infer the loglevel, but set it as a property so the subclasses can
# use it to set the loglevels on the loghandlers for their implementation
self._boto_log_level = LEVELS[log_level]
# this has to be 'public', so callers can use it. It's unfortunately
# near impossible to transparently wrap this, because the boto.config
# is initialized before we get here, and all the classes do a look up
# at compile time. So overriding doesn't help.
# Wrapping doesn't work cleanly, because we 1) would have to wrap
# everything, including future features we can't know about yet, as
# well as 2) poke into the guts of the implementation classes to figure
# out connection strings etc. It's quite cumbersome.
# So for now, we just store the region that was asked for, and let the
# caller use it. See the sample app for a howto.
self.cli_region = region
# if these are set, make sure we set the environment again
# as well; that way the underlying boto calls will just DTRT
# without the need to wrap all the functions.
credential_map = {
ACCESS_KEY: access_key,
SECRET_KEY: secret_key,
}
for env_var, val in iteritems(credential_map):
if val is None or len(val) < 1:
self._logger.debug(
'Passed boto credentials is empty. Falling back to environment variable %s',
env_var)
else:
# this way we can tell what credentials are being used,
# without dumping the whole secret into the logs
self._logger.debug('Setting boto credential %s', env_var)
os.environ[env_var] = val
# If at this point the environment variable is NOT set,
# you didn't set it, and we didn't set it. At which point
# boto will go off spelunking for .boto files or other
# settings. Best be clear about this. Using 'if not' because
# if you set it like this:
# $ FOO= ./myprog.py
# It'll return an empty string, and we'd not catch it.
if not os.environ.get(env_var, None):
self._logger.debug(
'Boto environment credential %s NOT explicitly set ' +
'-- boto will look for a .boto file somewhere', env_var)