def __init__(self, *args, **kwargs):
# Call to the superclass to resolve.
super(Boto, self).__init__(*args, **kwargs)
# access the boto classes via the object. Note these are just the
# classes for internal use, NOT the object as exposed via the CLI
# or the objects returned via the get_boto* calls
self._boto = boto
# This sets the log level for the underlying boto library
get_logger('boto').setLevel(self._boto_log_level)
def __init__(self, *args, **kwargs):
# Call to the superclass to resolve.
super(Boto, self).__init__(*args, **kwargs)
# access the boto classes via the object. Note these are just the
# classes for internal use, NOT the object as exposed via the CLI
# or the objects returned via the get_boto* calls
self._boto = boto
# This sets the log level for the underlying boto library
get_logger('boto').setLevel(self._boto_log_level)
def get_instance_region():
"""
Query the instance metadata service and return the region this instance is
placed in. If the metadata service can't be contacted, return a generic
default instead.
"""
# TODO: XXX This shouldn't get called if we're not on EC2.
zone = boto.utils.get_instance_metadata().get('placement', {}).get('availability-zone', None)
if zone is None:
get_logger('krux_boto').warn('get_instance_region failed to get the local instance region')
raise Error('get_instance_region failed to get the local instance region')
return zone.rstrip(string.lowercase)
def get_instance_region():
"""
Query the instance metadata service and return the region this instance is
placed in. If the metadata service can't be contacted, return a generic
default instead.
"""
# TODO: XXX This shouldn't get called if we're not on EC2.
zone = boto.utils.get_instance_metadata().get('placement', {}).get(
'availability-zone', None)
if zone is None:
get_logger('krux_boto').warn(
'get_instance_region failed to get the local instance region')
raise Error(
'get_instance_region failed to get the local instance region')
return zone.rstrip(string.lowercase)
def get_iam(args=None, logger=None, stats=None):
"""
Return a usable IAM object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones.
(If you omit the add_iam_cli_arguments() call during other cli setup,
the Boto object will still work, but its cli options won't show up in
--help output)
(This also handles instantiating a Boto3 object on its own.)
"""
if not args:
parser = get_parser(description=NAME)
add_iam_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
boto = Boto3(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
region=args.boto_region,
logger=logger,
stats=stats,
)
return IAM(
boto=boto,
logger=logger,
stats=stats,
)
def get_s3(args=None, logger=None, stats=None):
"""
Return a usable S3 object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones for the SQS setup.
(If you omit the add_s3_cli_arguments() call during other cli setup,
the Boto object will still work, but its cli options won't show up in
--help output)
(This also handles instantiating a Boto object on its own.)
"""
if not args:
parser = get_parser()
add_s3_cli_arguments(parser)
# Parse only the known arguments added by add_boto_cli_arguments().
# We only need those arguments to create Boto object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return S3(
boto=get_boto(args, logger, stats),
logger=logger,
stats=stats,
)
def get_cloud_formation(args=None, logger=None, stats=None):
"""
Return a usable CloudFormation object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones for the CloudFormation setup.
(If you omit the add_cloud_formation_cli_arguments() call during other cli setup,
the CloudFormation object will still work, but its cli options won't show up in
--help output)
(This also handles instantiating a Boto3 object on its own.)
"""
if not args:
parser = get_parser()
add_cloud_formation_cli_arguments(parser)
# Parse only the known arguments added by add_cloud_formation_cli_arguments().
# We only need those arguments to create CloudFormation object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
boto3 = Boto3(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
region=args.boto_region,
logger=logger,
stats=stats,
)
boto = Boto(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
# This boto is for S3 upload and is using a constant region,
# matching the TEMP_S3_BUCKET
region=getattr(args, 'bucket_region', CloudFormation.DEFAULT_S3_REGION),
logger=logger,
stats=stats,
)
s3 = S3(
boto=boto,
logger=logger,
stats=stats,
)
return CloudFormation(
boto=boto3,
s3=s3,
bucket_name=getattr(args, 'bucket_name', CloudFormation.DEFAULT_S3_BUCKET),
logger=logger,
stats=stats,
)
def __init__(
self,
boto,
s3,
bucket_name=DEFAULT_S3_BUCKET,
logger=None,
stats=None,
):
"""
:param boto: Boto3 object used to connect to Cloud Formation
:type boto: krux_boto.boto.Boto3
:param logger: Logger, recommended to be obtained using krux.cli.Application
:type logger: logging.Logger
:param stats: Stats, recommended to be obtained using krux.cli.Application
:type stats: kruxstatsd.StatsClient
"""
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
if not isinstance(boto, Boto3):
raise NotImplementedError(
'Currently krux_cloud_formation.cloud_formation.CloudFormation '
'only supports krux_boto.boto.Boto3'
)
self._s3 = s3
self._bucket_name = bucket_name
self._cf = boto.client('cloudformation')
self.template = troposphere.Template()
def get_boto3(args=None, logger=None, stats=None):
"""
Return a usable Boto3 object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If you don't have them,
however, we'll attempt to provide usable ones for the boto setup.
(If you omit the add_boto_cli_arguments() call during other cli setup,
the Boto object will still work, but its cli options won't show up in
--help output)
"""
if not args:
parser = get_parser()
add_boto_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return Boto3(
log_level=args.boto_log_level,
access_key=args.boto_access_key,
secret_key=args.boto_secret_key,
region=args.boto_region,
logger=logger,
stats=stats,
)
def __init__(self, logger=None, stats=None, parser=None):
self.master = None
self.slaves = []
self.name = "krux-redis"
self.logger = logger or get_logger(self.name)
self.stats = stats or get_stats(prefix=self.name)
self.parser = parser or get_parser(description=self.name)
### in case we got some of the information via the CLI
self.args = self.parser.parse_args()
def __init__(self, *args, **kwargs):
# Call to the superclass to resolve.
super(Boto3, self).__init__(*args, **kwargs)
# In boto3, the custom settings like region and connection params are
# stored in what's called a 'session'. This object behaves just like
# the boto3 class invocation, but it uses your custom settings instead.
# Read here for details: http://boto3.readthedocs.org/en/latest/guide/session.html
# Creating your own session, based on the region that was passed in
session = boto3.session.Session(region_name=self.cli_region)
# access the boto classes via the session. Note these are just the
# classes for internal use, NOT the object as exposed via the CLI
# or the objects returned via the get_boto* calls
self._boto = session
# This sets the log level for the underlying boto library
# http://boto3.readthedocs.org/en/latest/reference/core/boto3.html?highlight=logging
# XXX note that the name of the default boto3 logger is NOT boto3, it's
# called 'botocore'
get_logger('botocore').setLevel(self._boto_log_level)
def __init__(self, *args, **kwargs):
# Call to the superclass to resolve.
super(Boto3, self).__init__(*args, **kwargs)
# In boto3, the custom settings like region and connection params are
# stored in what's called a 'session'. This object behaves just like
# the boto3 class invocation, but it uses your custom settings instead.
# Read here for details: http://boto3.readthedocs.org/en/latest/guide/session.html
# Creating your own session, based on the region that was passed in
session = boto3.session.Session(region_name=self.cli_region)
# access the boto classes via the session. Note these are just the
# classes for internal use, NOT the object as exposed via the CLI
# or the objects returned via the get_boto* calls
self._boto = session
# This sets the log level for the underlying boto library
# http://boto3.readthedocs.org/en/latest/reference/core/boto3.html?highlight=logging
# XXX note that the name of the default boto3 logger is NOT boto3, it's
# called 'botocore'
get_logger('botocore').setLevel(self._boto_log_level)
def __init__(
self,
hostname,
use_ssl=True,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
self._hostname = hostname
self._protocol = 'https' if use_ssl else 'http'
def __init__(
self,
boto,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
# Private client representing IAM
self._client = boto.client(IAM._IAM_STR)
def __init__(
self,
boto,
name=NAME,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = name
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
self._boto = boto
self._listeners = []
def __init__(self, name=None, logger=None, stats=None):
"""
Basic init method that sets up name, logger, and stats
:param name: Name of the application
:type name: str
:param logger: Logger, recommended to be obtained using krux.cli.Application
:type logger: logging.Logger
:param stats: Stats, recommended to be obtained using krux.cli.Application
:type stats: kruxstatsd.StatsClient
"""
# Private variables, not to be used outside this module
self._name = name if name is not None else self.__class__.__name__
self._logger = logger if logger is not None else get_logger(self._name)
self._stats = stats if stats is not None else get_stats(prefix=self._name)
def __init__(
self,
boto,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
if not isinstance(boto, Boto3):
raise NotImplementedError('Currently krux_boto.sqs.Sqs only supports krux_boto.boto.Boto3')
self._resource = boto.resource('sqs')
self._queues = {}
def __init__(
self,
flow_token,
name=NAME,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = name
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
self._flowdock = flowdock.Chat(flow_token)
self._regular_events = []
self._urgent_events = []
self.urgent_threshold = self.DEFAULT_URGENT_THRESHOLD_HOURS
def get_cloud_health(args=None, logger=None, stats=None):
if not args:
parser = get_parser(description=NAME)
add_cloud_health_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return CloudHealth(
api_key=args.api_key,
logger=logger,
stats=stats,
)
def get_cloud_health(args=None, logger=None, stats=None):
if not args:
parser = get_parser(description=NAME)
add_cloud_health_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return CloudHealth(
api_key=args.api_key,
logger=logger,
stats=stats,
)
def __init__(
self,
username,
password,
base_url=KRUX_JIRA_URL,
name=NAME,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
self._headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
}
self._auth = (username, password)
self._base_url = base_url
def __init__(
self,
boto,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
# Throw exception when Boto2 is not used
# TODO: Start using Boto3 and reverse this check
if not isinstance(boto, Boto):
raise TypeError('krux_elb.elb.ELB only supports krux_boto.boto.Boto')
self.boto = boto
# Set up default cache
self._conn = None
def __init__(self, logger = None, stats = None, parser = None, application = None):
"""
Either pass in a krux.cli.Application, or separate stats/logger/parser instances.
If you pass in none of those, we'll create new instances here.
If you pass the Application class, exit hooks can be run if you ask the scheduler
to exit after its job completes. This is recommended.
"""
### to keep line length short below
_a = application
self.application = application
self.name = getattr(_a, 'name', 'krux-scheduler')
self.logger = logger or getattr(_a, 'logger', get_logger(self.name))
self.stats = stats or getattr(_a, 'stats', get_stats( prefix = self.name))
self.parser = parser or getattr(_a, 'parser', get_parser(description = self.name))
### we keep track of the last job that was run, so we can use that
### as part of the exit status in the App.
self.last_job_event = None
### in case we got some of the information via the CLI
self.args = self.parser.parse_args()
### Own process information - always record this when we have
### a chance to do so.
self.proc = psutil.Process(os.getpid())
### the AP scheduler object
_standalone = True if not self.args.scheduler_daemonize else False
self.___apscheduler = apscheduler.scheduler.Scheduler(
standalone = _standalone,
daemonic = not _standalone,
)
### catch any events that come up
self.___setup_event_listener(
terminate_on_finish = self.args.scheduler_exit_after_job,
)
def __get_arguments(args=None, logger=None, stats=None):
"""
A helper method that generates a dictionary of arguments needed to instantiate a BaseBoto object.
The purpose of this method is to abstract out the code to handle optional CLI arguments
and not duplicate the None handling code.
:param args: Namespace of arguments parsed by argparse
:type args: argparse.Namespace
:param logger: Logger, recommended to be obtained using krux.cli.Application
:type logger: logging.Logger
:param stats: Stats, recommended to be obtained using krux.cli.Application
:type stats: kruxstatsd.StatsClient
:return: A dictionary of arguments needed for BaseBoto.__init__()
:rtype: dict
"""
if not args:
parser = get_parser()
add_boto_cli_arguments(parser)
# Parse only the known arguments added by add_boto_cli_arguments().
# We only need those arguments to create Boto object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return {
'log_level': getattr(args, 'boto_log_level', DEFAULT['log_level']()),
'access_key': getattr(args, 'boto_access_key',
DEFAULT['access_key']()),
'secret_key': getattr(args, 'boto_secret_key',
DEFAULT['secret_key']()),
'region': getattr(args, 'boto_region', DEFAULT['region']()),
'logger': logger,
'stats': stats,
}
def __get_arguments(args=None, logger=None, stats=None):
"""
A helper method that generates a dictionary of arguments needed to instantiate a BaseBoto object.
The purpose of this method is to abstract out the code to handle optional CLI arguments
and not duplicate the None handling code.
:param args: Namespace of arguments parsed by argparse
:type args: argparse.Namespace
:param logger: Logger, recommended to be obtained using krux.cli.Application
:type logger: logging.Logger
:param stats: Stats, recommended to be obtained using krux.cli.Application
:type stats: kruxstatsd.StatsClient
:return: A dictionary of arguments needed for BaseBoto.__init__()
:rtype: dict
"""
if not args:
parser = get_parser()
add_boto_cli_arguments(parser)
# Parse only the known arguments added by add_boto_cli_arguments().
# We only need those arguments to create Boto object, nothing else.
# parse_known_args() return (Namespace, list of unknown arguments),
# we only care about the Namespace object here.
args = parser.parse_known_args()[0]
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return {
'log_level': getattr(args, 'boto_log_level', DEFAULT['log_level']()),
'access_key': getattr(args, 'boto_access_key', DEFAULT['access_key']()),
'secret_key': getattr(args, 'boto_secret_key', DEFAULT['secret_key']()),
'region': getattr(args, 'boto_region', DEFAULT['region']()),
'logger': logger,
'stats': stats,
}
def get_kafka_manager_api(args=None, logger=None, stats=None):
"""
Return a usable Kafka Manager object without creating a class around it.
In the context of a krux.cli (or similar) interface the 'args', 'logger'
and 'stats' objects should already be present. If they are not inputted,
we will provide usable ones.
"""
if not args:
parser = get_parser(description=NAME)
add_kafka_manager_api_cli_arguments(parser)
args = parser.parse_args()
if not logger:
logger = get_logger(name=NAME)
if not stats:
stats = get_stats(prefix=NAME)
return KafkaManager(
hostname=args.hostname,
use_ssl=args.use_ssl,
logger=logger,
stats=stats,
)
def __init__(
self,
log_level=None,
access_key=None,
secret_key=None,
region=None,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
if log_level is None:
log_level = DEFAULT['log_level']()
if access_key is None:
access_key = DEFAULT['access_key']()
if secret_key is None:
secret_key = DEFAULT['secret_key']()
if region is None:
region = DEFAULT['region']()
if REGION not in os.environ:
self._logger.debug(
"There is not a default region set in your environment variables. Defaulted to '%s'", region
)
# GOTCHA: Due to backward incompatible version change in v1.0.0, the users of krux_boto may
# pass wrong credential. Make sure the passed credential via CLI is the same as one passed into this instance.
parser = get_parser()
add_boto_cli_arguments(parser)
# GOTCHA: We only care about the credential arguments and nothing else.
# Don't validate the arguments or parse other things. Let krux.cli do that.
args = parser.parse_known_args()
_access_key = getattr(args, 'boto_access_key', None)
_secret_key = getattr(args, 'boto_secret_key', None)
if _access_key is not None and _access_key != access_key:
self._logger.warn(
'You set %s as boto-access-key in CLI, but passed %s to the library. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.',
BaseBoto._hide_value(_access_key), BaseBoto._hide_value(access_key),
)
if _secret_key is not None and _secret_key != secret_key:
self._logger.warn(
'You set %s as boto-secret-key in CLI, but passed %s to the library. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.',
BaseBoto._hide_value(_secret_key), BaseBoto._hide_value(secret_key),
)
# Infer the loglevel, but set it as a property so the subclasses can
# use it to set the loglevels on the loghandlers for their implementation
self._boto_log_level = LEVELS[log_level]
# this has to be 'public', so callers can use it. It's unfortunately
# near impossible to transparently wrap this, because the boto.config
# is initialized before we get here, and all the classes do a look up
# at compile time. So overriding doesn't help.
# Wrapping doesn't work cleanly, because we 1) would have to wrap
# everything, including future features we can't know about yet, as
# well as 2) poke into the guts of the implementation classes to figure
# out connection strings etc. It's quite cumbersome.
# So for now, we just store the region that was asked for, and let the
# caller use it. See the sample app for a howto.
self.cli_region = region
# if these are set, make sure we set the environment again
# as well; that way the underlying boto calls will just DTRT
# without the need to wrap all the functions.
credential_map = {
ACCESS_KEY: access_key,
SECRET_KEY: secret_key,
}
for env_var, val in iteritems(credential_map):
if val is None or len(val) < 1:
self._logger.debug('Passed boto credentials is empty. Falling back to environment variable %s', env_var)
else:
# this way we can tell what credentials are being used,
# without dumping the whole secret into the logs
self._logger.debug('Setting boto credential %s to %s', env_var, BaseBoto._hide_value(val))
os.environ[env_var] = val
# If at this point the environment variable is NOT set,
# you didn't set it, and we didn't set it. At which point
# boto will go off spelunking for .boto files or other
# settings. Best be clear about this. Using 'if not' because
# if you set it like this:
# $ FOO= ./myprog.py
# It'll return an empty string, and we'd not catch it.
if not os.environ.get(env_var, None):
self._logger.debug(
'Boto environment credential %s NOT explicitly set ' +
'-- boto will look for a .boto file somewhere', env_var
)
def __init__(
self,
log_level=None,
access_key=None,
secret_key=None,
region=None,
logger=None,
stats=None,
):
# Private variables, not to be used outside this module
self._name = NAME
self._logger = logger or get_logger(self._name)
self._stats = stats or get_stats(prefix=self._name)
if log_level is None:
log_level = DEFAULT['log_level']()
if access_key is None:
access_key = DEFAULT['access_key']()
if secret_key is None:
secret_key = DEFAULT['secret_key']()
if region is None:
region = DEFAULT['region']()
if REGION not in os.environ:
self._logger.debug(
"There is not a default region set in your environment variables. Defaulted to '%s'",
region)
# GOTCHA: Due to backward incompatible version change in v1.0.0, the users of krux_boto may
# pass wrong credential. Make sure the passed credential via CLI is the same as one passed into this instance.
parser = get_parser()
add_boto_cli_arguments(parser)
# GOTCHA: We only care about the credential arguments and nothing else.
# Don't validate the arguments or parse other things. Let krux.cli do that.
args = parser.parse_known_args()
_access_key = getattr(args, 'boto_access_key', None)
_secret_key = getattr(args, 'boto_secret_key', None)
if _access_key is not None and _access_key != access_key:
self._logger.warn(
'You set a different boto-access-key in CLI. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.')
if _secret_key is not None and _secret_key != secret_key:
self._logger.warn(
'You set a different boto-secret-key in CLI. '
'To avoid this error, consider using get_boto() function. '
'For more information, please check README.')
# Infer the loglevel, but set it as a property so the subclasses can
# use it to set the loglevels on the loghandlers for their implementation
self._boto_log_level = LEVELS[log_level]
# this has to be 'public', so callers can use it. It's unfortunately
# near impossible to transparently wrap this, because the boto.config
# is initialized before we get here, and all the classes do a look up
# at compile time. So overriding doesn't help.
# Wrapping doesn't work cleanly, because we 1) would have to wrap
# everything, including future features we can't know about yet, as
# well as 2) poke into the guts of the implementation classes to figure
# out connection strings etc. It's quite cumbersome.
# So for now, we just store the region that was asked for, and let the
# caller use it. See the sample app for a howto.
self.cli_region = region
# if these are set, make sure we set the environment again
# as well; that way the underlying boto calls will just DTRT
# without the need to wrap all the functions.
credential_map = {
ACCESS_KEY: access_key,
SECRET_KEY: secret_key,
}
for env_var, val in iteritems(credential_map):
if val is None or len(val) < 1:
self._logger.debug(
'Passed boto credentials is empty. Falling back to environment variable %s',
env_var)
else:
# this way we can tell what credentials are being used,
# without dumping the whole secret into the logs
self._logger.debug('Setting boto credential %s', env_var)
os.environ[env_var] = val
# If at this point the environment variable is NOT set,
# you didn't set it, and we didn't set it. At which point
# boto will go off spelunking for .boto files or other
# settings. Best be clear about this. Using 'if not' because
# if you set it like this:
# $ FOO= ./myprog.py
# It'll return an empty string, and we'd not catch it.
if not os.environ.get(env_var, None):
self._logger.debug(
'Boto environment credential %s NOT explicitly set ' +
'-- boto will look for a .boto file somewhere', env_var)
