def thread(self, function, *args, **kwargs):
"""Execute `function` in a concurrent thread.
Example:
>>> zones = yield thread(ec2_conn.get_all_zones)
This allows execution of any function in a thread without having
to write a wrapper method that is decorated with run_on_executor()
"""
try:
return function(*args, **kwargs)
except boto_exception.BotoServerError as e:
# If we're using temporary IAM credentials, when those expire we
# can get back a blank 400 from Amazon. This is confusing, but it
# happens because of https://github.com/boto/boto/issues/898. In
# most cases, these temporary IAM creds can be re-loaded by
# reaching out to the AWS API (for example, if we're using an IAM
# Instance Profile role), so thats what Boto tries to do. However,
# if you're using short-term creds (say from SAML auth'd logins),
# then this fails and Boto returns a blank 400.
if (e.status == 400 and e.reason == 'Bad Request'
and e.error_code is None):
msg = 'Access credentials have expired'
raise exceptions.InvalidCredentials(msg)
msg = '%s: %s' % (e.error_code, e.message)
if e.status == 403:
raise exceptions.InvalidCredentials(msg)
raise
except boto3_exceptions.Boto3Error as e:
raise exceptions.RecoverableActorFailure(
'Boto3 had a failure: %s' % e)
def __init__(self, *args, **kwargs):
super(SpotinstBase, self).__init__(*args, **kwargs)
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "SPOTINST_TOKEN" environment variable.')
if not DEBUG:
logging.getLogger('tornado_rest_client.api').setLevel('INFO')
# Figure out our account ID and set it.. Or this will end up falling
# back to None if neither are set.
account_id = self._options.get('account_id')
if account_id is None:
account_id = ACCOUNT_ID
if account_id is None:
raise exceptions.InvalidCredentials(
'Missing SPOTINST_ACCOUNT_ID or account_id parameter')
rest_client = SpotinstRestClient(
headers={
'Authorization': 'Bearer %s' % TOKEN,
'Content-Type': 'application/json',
})
self._client = SpotinstAPI(client=rest_client, account_id=account_id)
def __init__(self, *args, **kwargs):
"""Check for the needed environment variables."""
super(Annotation, self).__init__(*args, **kwargs)
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "LIBRATO_TOKEN" environment variable.')
if not EMAIL:
raise exceptions.InvalidCredentials(
'Missing the "LIBRATO_EMAIL" environment variable.')
def __init__(self, *args, **kwargs):
"""Check required environment variables."""
super(PackagecloudBase, self).__init__(*args, **kwargs)
if not ACCOUNT:
raise exceptions.InvalidCredentials(
'Missing the "PACKAGECLOUD_ACCOUNT" environment variable.')
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "PACKAGECLOUD_TOKEN" environment variable.')
rest_client = api.RestClient(timeout=120)
self._packagecloud_client = PackagecloudAPI(client=rest_client)
def _fetch_wrapper(self, *args, **kwargs):
"""Wrap the superclass _fetch method to catch known Rollbar errors.
https://rollbar.com/docs/api_overview/
"""
try:
res = yield self._fetch(*args, **kwargs)
except httpclient.HTTPError as e:
# These are HTTPErrors that we know about, and can log specific
# error messages for.
if e.code in (401, 403):
raise exceptions.InvalidCredentials(
'The "ROLLBAR_TOKEN" is invalid')
elif e.code == 422:
raise exceptions.RecoverableActorFailure(
'Unprocessable Entity - the request was parseable (i.e. '
'valid JSON), but some parameters were missing or '
'otherwise invalid.')
elif e.code == 429:
raise exceptions.RecoverableActorFailure(
'Too Many Requests - If rate limiting is enabled for '
'your access token, this return code signifies that the '
'rate limit has been reached and the item was not '
'processed.')
else:
# We ran into a problem we can't handle. Also, keep in mind
# that @utils.retry() was used, so this error happened several
# times before getting here. Raise it.
raise exceptions.RecoverableActorFailure(
'Unexpected error from Rollbar API: %s' % e)
raise gen.Return(res)
def __init__(self, *args, **kwargs):
"""Check for required settings."""
super(AWSBaseActor, self).__init__(*args, **kwargs)
if not (aws_settings.AWS_ACCESS_KEY_ID
and aws_settings.AWS_SECRET_ACCESS_KEY):
raise exceptions.InvalidCredentials(
'AWS settings imported but not all credentials are supplied. '
'AWS_ACCESS_KEY_ID: %s, AWS_SECRET_ACCESS_KEY: %s' %
(aws_settings.AWS_ACCESS_KEY_ID,
aws_settings.AWS_SECRET_ACCESS_KEY))
# Establish connection objects that don't require a region
self.iam_conn = boto.iam.connection.IAMConnection(
aws_access_key_id=aws_settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=aws_settings.AWS_SECRET_ACCESS_KEY)
# Establish region-specific connection objects.
region = self.option('region')
if not region:
return
# In case a zone was provided instead of region we can convert
# it on the fly
zone_check = re.match(r'(.*[0-9])([a-z]*)$', region)
if zone_check and zone_check.group(2):
zone = region # Only saving this for the log below
# Set the fixed region
region = zone_check.group(1)
self.log.warning('Converting zone "%s" to region "%s".' %
(zone, region))
region_names = [r.name for r in boto.ec2.elb.regions()]
if region not in region_names:
err = ('Region "%s" not found. Available regions: %s' %
(region, region_names))
raise exceptions.InvalidOptions(err)
self.ec2_conn = boto.ec2.connect_to_region(
region,
aws_access_key_id=aws_settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=aws_settings.AWS_SECRET_ACCESS_KEY)
self.elb_conn = boto.ec2.elb.connect_to_region(
region,
aws_access_key_id=aws_settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=aws_settings.AWS_SECRET_ACCESS_KEY)
self.cf_conn = boto.cloudformation.connect_to_region(
region,
aws_access_key_id=aws_settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=aws_settings.AWS_SECRET_ACCESS_KEY)
self.sqs_conn = boto.sqs.connect_to_region(
region,
aws_access_key_id=aws_settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=aws_settings.AWS_SECRET_ACCESS_KEY)
def __init__(self, *args, **kwargs):
"""Initializes the Actor."""
super(RightScaleBaseActor, self).__init__(*args, **kwargs)
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "RIGHTSCALE_TOKEN" environment variable.')
self._client = api.RightScale(token=TOKEN, endpoint=ENDPOINT)
def __init__(self, *args, **kwargs):
"""Check required environment variables."""
super(RollbarBase, self).__init__(*args, **kwargs)
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "ROLLBAR_TOKEN" environment variable.')
self._token = TOKEN
def __init__(self, *args, **kwargs):
"""Check required environment variables."""
super(SlackBase, self).__init__(*args, **kwargs)
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "SLACK_TOKEN" environment variable.')
rest_client = api.SimpleTokenRestClient(tokens={'token': TOKEN})
self._slack_client = SlackAPI(client=rest_client)
def __init__(self, *args, **kwargs):
"""Check required environment variables."""
super(HipchatBase, self).__init__(*args, **kwargs)
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "HIPCHAT_TOKEN" environment variable.')
self._token = TOKEN
self._name = self._validate_from_name(NAME)
def _wrap_boto_exception(self, e):
if isinstance(e, boto_exception.BotoServerError):
# If we're using temporary IAM credentials, when those expire we
# can get back a blank 400 from Amazon. This is confusing, but it
# happens because of https://github.com/boto/boto/issues/898. In
# most cases, these temporary IAM creds can be re-loaded by
# reaching out to the AWS API (for example, if we're using an IAM
# Instance Profile role), so thats what Boto tries to do. However,
# if you're using short-term creds (say from SAML auth'd logins),
# then this fails and Boto returns a blank 400.
if (e.status == 400 and e.reason == 'Bad Request'
and e.error_code is None):
msg = 'Access credentials have expired'
return exceptions.InvalidCredentials(msg)
msg = '%s: %s' % (e.error_code, str(e))
if e.status == 403:
return exceptions.InvalidCredentials(msg)
elif isinstance(e, boto3_exceptions.Boto3Error):
return exceptions.RecoverableActorFailure(
'Boto3 had a failure: %s' % e)
return e
def _fetch_wrapper(self, *args, **kwargs):
"""Wrap the superclass _fetch method to catch known Librato errors."""
try:
res = yield self._fetch(*args, **kwargs)
except httpclient.HTTPError as e:
if e.code == 400:
# "HTTPError: HTTP 400: Bad Request"
raise exceptions.BadRequest('Check your JSON inputs.')
if e.code == 401:
# "The authentication you provided is invalid."
raise exceptions.InvalidCredentials(
'Librato authentication failed.')
raise
raise gen.Return(res)
def __init__(self, *args, **kwargs):
super(SpotinstBase, self).__init__(*args, **kwargs)
if not TOKEN:
raise exceptions.InvalidCredentials(
'Missing the "SPOTINST_TOKEN" environment variable.')
if not DEBUG:
logging.getLogger('tornado_rest_client.api').setLevel('INFO')
rest_client = SpotinstRestClient(
headers={
'Authorization': 'Bearer %s' % TOKEN,
'Content-Type': 'application/json',
})
self._client = SpotinstAPI(client=rest_client)
def thread(self, function, *args, **kwargs):
"""Execute `function` in a concurrent thread.
Example:
>>> zones = yield thread(ec2_conn.get_all_zones)
This allows execution of any function in a thread without having
to write a wrapper method that is decorated with run_on_executor()
"""
try:
return function(*args, **kwargs)
except boto_exception.BotoServerError as e:
msg = '%s: %s' % (e.error_code, e.message)
if e.status == 403:
raise exceptions.InvalidCredentials(msg)
raise
def _check_access(self, elb):
"""Perform a dummy operation to check credential accesss.
Intended to be used in a dry run, this method attempts to perform an
invalid set_listener call and monitors the output of the error. If the
error is anything other than AccessDenied then the provided credentials
are sufficient and we do nothing.
Args:
elb: boto LoadBalancer object.
"""
try:
# A blank ARN value should have code 'CertificateNotFound'
# We're only checking if credentials have sufficient access
yield self.api_call(elb.set_listener_SSL_certificate,
self.option('port'), '')
except BotoServerError as e:
if e.error_code == 'AccessDenied':
raise exceptions.InvalidCredentials(e)
def _execute(self):
if self._dry:
raise gen.Return(self._execute_dry())
# Only generate a JSON text string if a populated dict was passed to
# data-json.
datajson = None
if self.option('data-json'):
datajson = json.dumps(self.option('data-json'))
escaped_post = (urllib.parse.urlencode(self.option('data')) or datajson
or None)
try:
yield self._fetch(self.option('url'),
post=escaped_post,
auth_username=self.option('username'),
auth_password=self.option('password'))
except httpclient.HTTPError as e:
if e.code == 401:
raise exceptions.InvalidCredentials(e.message)
def _fetch_wrapper(self, *args, **kwargs):
"""Wrap the superclass _fetch method to catch known Hipchat errors."""
try:
res = yield self._fetch(*args, **kwargs)
except httpclient.HTTPError as e:
# These are HTTPErrors that we know about, and can log specific
# error messages for.
self.log.critical(e)
if e.code in (401, 403):
# "The authentication you provided is invalid."
raise exceptions.InvalidCredentials(
'The "HIPCHAT_NAME" or "HIPCHAT_TOKEN" supplied is '
'invalid. %s' % e)
else:
# We ran into a problem we can't handle. Also, keep in mind
# that @utils.retry() was used, so this error happened several
# times before getting here. Raise it.
raise exceptions.RecoverableActorFailure(
'Unexpected error from Hipchat API: %s' % e)
raise gen.Return(res)
def __init__(self, *args, **kwargs):
"""Check for required settings."""
super(AWSBaseActor, self).__init__(*args, **kwargs)
# By default, we will try to let Boto handle discovering its
# credentials at instantiation time. This _can_ result in synchronous
# API calls to the Metadata service, but those should be fast.
key = None
secret = None
# In the event though that someone has explicitly set the AWS access
# keys in the environment (either for the purposes of a unit test, or
# because they wanted to), we use those values.
if (aws_settings.AWS_ACCESS_KEY_ID
and aws_settings.AWS_SECRET_ACCESS_KEY):
key = aws_settings.AWS_ACCESS_KEY_ID
secret = aws_settings.AWS_SECRET_ACCESS_KEY
# On our first simple IAM connection, test the credentials and make
# sure things worked!
try:
# Establish connection objects that don't require a region
self.iam_conn = boto.iam.connection.IAMConnection(
aws_access_key_id=key, aws_secret_access_key=secret)
except boto.exception.NoAuthHandlerFound:
raise exceptions.InvalidCredentials(
'AWS settings imported but not all credentials are supplied. '
'AWS_ACCESS_KEY_ID: %s, AWS_SECRET_ACCESS_KEY: %s' %
(aws_settings.AWS_ACCESS_KEY_ID,
aws_settings.AWS_SECRET_ACCESS_KEY))
# Establish region-specific connection objects.
region = self.option('region')
if not region:
return
# In case a zone was provided instead of region we can convert
# it on the fly
zone_check = re.match(r'(.*[0-9])([a-z]*)$', region)
if zone_check and zone_check.group(2):
zone = region # Only saving this for the log below
# Set the fixed region
region = zone_check.group(1)
self.log.warning('Converting zone "%s" to region "%s".' %
(zone, region))
region_names = [r.name for r in boto.ec2.elb.regions()]
if region not in region_names:
err = ('Region "%s" not found. Available regions: %s' %
(region, region_names))
raise exceptions.InvalidOptions(err)
self.ec2_conn = boto.ec2.connect_to_region(
region, aws_access_key_id=key, aws_secret_access_key=secret)
self.ecs_conn = boto3.client('ecs',
region_name=region,
aws_access_key_id=key,
aws_secret_access_key=secret)
self.elb_conn = boto.ec2.elb.connect_to_region(
region, aws_access_key_id=key, aws_secret_access_key=secret)
self.elbv2_conn = boto3.client('elbv2',
region_name=region,
aws_access_key_id=key,
aws_secret_access_key=secret)
self.cf3_conn = boto3.client('cloudformation',
region_name=region,
aws_access_key_id=key,
aws_secret_access_key=secret)
self.sqs_conn = boto.sqs.connect_to_region(
region, aws_access_key_id=key, aws_secret_access_key=secret)
self.s3_conn = boto3.client('s3',
region_name=region,
aws_access_key_id=key,
aws_secret_access_key=secret)
