def build_list_params(self, params, items, label):
"""
Items is a list of dictionaries or strings::
[
{
'Protocol' : 'HTTP',
'LoadBalancerPort' : '80',
'InstancePort' : '80'
},
..
] etc.
or::
['us-east-1b',...]
"""
# different from EC2 list params
for i in range(1, len(items) + 1):
if isinstance(items[i - 1], dict):
for k, v in six.iteritems(items[i - 1]):
if isinstance(v, dict):
for kk, vv in six.iteritems(v):
params['%s.member.%d.%s.%s' % (label, i, k, kk)] = vv
else:
params['%s.member.%d.%s' % (label, i, k)] = v
elif isinstance(items[i - 1], six.string_types):
params['%s.member.%d' % (label, i)] = items[i - 1]
def to_domain_connection_params(self):
"""
Transform search parameters from instance properties to a dictionary
that CloudSearchDomainConnection can accept
:rtype: dict
:return: search parameters
"""
params = {'start': self.start, 'size': self.real_size}
if self.q:
params['q'] = self.q
if self.parser:
params['query_parser'] = self.parser
if self.fq:
params['filter_query'] = self.fq
if self.expr:
expr = {}
for k, v in six.iteritems(self.expr):
expr['expr.%s' % k] = v
params['expr'] = expr
if self.facet:
facet = {}
for k, v in six.iteritems(self.facet):
if not isinstance(v, six.string_types):
v = json.dumps(v)
facet['facet.%s' % k] = v
params['facet'] = facet
if self.highlight:
highlight = {}
for k, v in six.iteritems(self.highlight):
highlight['highlight.%s' % k] = v
params['highlight'] = highlight
if self.options:
params['query_options'] = self.options
if self.return_fields:
params['ret'] = ','.join(self.return_fields)
if self.partial is not None:
params['partial'] = self.partial
if self.sort:
params['sort'] = ','.join(self.sort)
return params
def resubmit(self):
"""
Resubmit the batch to get the next result set. The request object is
rebuild from scratch meaning that all batch added between ``submit``
and ``resubmit`` will be lost.
Note: This method is experimental and subject to changes in future releases
"""
del self[:]
if not self.unprocessed:
return None
for table_name, table_req in six.iteritems(self.unprocessed):
table_keys = table_req['Keys']
table = self.layer2.get(table_name)
keys = []
for key in table_keys:
h = key['HashKeyElement']
r = None
if 'RangeKeyElement' in key:
r = key['RangeKeyElement']
keys.append((h, r))
attributes_to_get = None
if 'AttributesToGet' in table_req:
attributes_to_get = table_req['AttributesToGet']
self.add_batch(table, keys, attributes_to_get=attributes_to_get)
return self.submit()
def _build_step_list(self, steps):
if not isinstance(steps, list):
steps = [steps]
params = {}
for i, step in enumerate(steps):
for key, value in six.iteritems(step):
params['Steps.member.%s.%s' % (i+1, key)] = value
return params
def _build_bootstrap_action_list(self, bootstrap_actions):
if not isinstance(bootstrap_actions, list):
bootstrap_actions = [bootstrap_actions]
params = {}
for i, bootstrap_action in enumerate(bootstrap_actions):
for key, value in six.iteritems(bootstrap_action):
params['BootstrapActions.member.%s.%s' % (i + 1, key)] = value
return params
def to_params(self):
"""Transform search parameters from instance properties to a dictionary
:rtype: dict
:return: search parameters
"""
params = {"start": self.start, "size": self.real_size}
if self.q:
params["q"] = self.q
if self.parser:
params["q.parser"] = self.parser
if self.fq:
params["fq"] = self.fq
if self.expr:
for k, v in six.iteritems(self.expr):
params["expr.%s" % k] = v
if self.facet:
for k, v in six.iteritems(self.facet):
if type(v) not in [str, unicode]:
v = json.dumps(v)
params["facet.%s" % k] = v
if self.highlight:
for k, v in six.iteritems(self.highlight):
params["highlight.%s" % k] = v
if self.options:
params["options"] = self.options
if self.return_fields:
params["return"] = ",".join(self.return_fields)
if self.partial is not None:
params["partial"] = self.partial
if self.sort:
params["sort"] = ",".join(self.sort)
return params
def to_params(self):
"""Transform search parameters from instance properties to a dictionary
:rtype: dict
:return: search parameters
"""
params = {'start': self.start, 'size': self.real_size}
if self.q:
params['q'] = self.q
if self.parser:
params['q.parser'] = self.parser
if self.fq:
params['fq'] = self.fq
if self.expr:
for k, v in six.iteritems(self.expr):
params['expr.%s' % k] = v
if self.facet:
for k, v in six.iteritems(self.facet):
if not isinstance(v, six.string_types):
v = json.dumps(v)
params['facet.%s' % k] = v
if self.highlight:
for k, v in six.iteritems(self.highlight):
params['highlight.%s' % k] = v
if self.options:
params['options'] = self.options
if self.return_fields:
params['return'] = ','.join(self.return_fields)
if self.partial is not None:
params['partial'] = self.partial
if self.sort:
params['sort'] = ','.join(self.sort)
return params
def make_request(self, action, path, headers=None, data='', params=None):
if params:
pairs = []
for key, val in six.iteritems(params):
if val is None:
continue
pairs.append(key + '=' + urllib.parse.quote(str(val)))
path += '?' + '&'.join(pairs)
return super(Route53Connection, self).make_request(
action, path, headers, data, retry_handler=self._retry_handler)
def to_params(self):
"""Transform search parameters from instance properties to a dictionary
:rtype: dict
:return: search parameters
"""
params = {'start': self.start, 'size': self.real_size}
if self.q:
params['q'] = self.q
if self.bq:
params['bq'] = self.bq
if self.rank_expressions:
for k, v in self.rank_expressions.iteritems():
params['rank-%s' % k] = v
if self.rank:
params['rank'] = ','.join(self.rank)
if self.return_fields:
params['return-fields'] = ','.join(self.return_fields)
if self.facet:
params['facet'] = ','.join(self.facet)
if self.facet_constraints:
for k, v in six.iteritems(self.facet_constraints):
params['facet-%s-constraints' % k] = v
if self.facet_sort:
for k, v in six.iteritems(self.facet_sort):
params['facet-%s-sort' % k] = v
if self.facet_top_n:
for k, v in six.iteritems(self.facet_top_n):
params['facet-%s-top-n' % k] = v
if self.t:
for k, v in six.iteritems(self.t):
params['t-%s' % k] = v
return params
def _build_tag_list(self, tags):
assert isinstance(tags, dict)
params = {}
for i, key_value in enumerate(sorted(six.iteritems(tags)), start=1):
key, value = key_value
current_prefix = 'Tags.member.%s' % i
params['%s.Key' % current_prefix] = key
if value:
params['%s.Value' % current_prefix] = value
return params
def build_list_params(self, params, items, label):
if isinstance(items, six.string_types):
items = [items]
for index, item in enumerate(items):
i = index + 1
if isinstance(item, dict):
for k, v in six.iteritems(item):
params[label % (i, 'Name')] = k
if v is not None:
params[label % (i, 'Value')] = v
else:
params[label % i] = item
def __repr__(self):
result = self.__class__.__name__ + '{ '
counter = 0
for key, value in six.iteritems(self.__dict__):
# first iteration no comma
counter += 1
if counter > 1:
result += ', '
result += key + ': '
result += self._repr_by_type(value)
result += ' }'
return result
def create_lb_policy(self, lb_name, policy_name, policy_type, policy_attributes):
"""
Creates a new policy that contains the necessary attributes
depending on the policy type. Policies are settings that are
saved for your load balancer and that can be applied to the
front-end listener, or the back-end application server.
"""
params = {"LoadBalancerName": lb_name, "PolicyName": policy_name, "PolicyTypeName": policy_type}
for index, (name, value) in enumerate(six.iteritems(policy_attributes), 1):
params["PolicyAttributes.member.%d.AttributeName" % index] = name
params["PolicyAttributes.member.%d.AttributeValue" % index] = value
else:
params["PolicyAttributes"] = ""
return self.get_status("CreateLoadBalancerPolicy", params)
def _build_instance_group_list_args(self, instance_groups):
"""
Takes a list of InstanceGroups, or a single InstanceGroup. Returns
a comparable dict for use in making a RunJobFlow or AddInstanceGroups
request.
"""
if not isinstance(instance_groups, list):
instance_groups = [instance_groups]
params = {}
for i, instance_group in enumerate(instance_groups):
ig_dict = self._build_instance_group_args(instance_group)
for key, value in six.iteritems(ig_dict):
params['InstanceGroups.member.%d.%s' % (i + 1, key)] = value
return params
def _build_instance_group_list_args(self, instance_groups):
"""
Takes a list of InstanceGroups, or a single InstanceGroup. Returns
a comparable dict for use in making a RunJobFlow or AddInstanceGroups
request.
"""
if not isinstance(instance_groups, list):
instance_groups = [instance_groups]
params = {}
for i, instance_group in enumerate(instance_groups):
ig_dict = self._build_instance_group_args(instance_group)
for key, value in six.iteritems(ig_dict):
params['InstanceGroups.member.%d.%s' % (i+1, key)] = value
return params
def get_key(self,
key_name,
headers=None,
version_id=None,
response_headers=None,
generation=None):
"""Returns a Key instance for an object in this bucket.
Note that this method uses a HEAD request to check for the existence of
the key.
:type key_name: string
:param key_name: The name of the key to retrieve
:type response_headers: dict
:param response_headers: A dictionary containing HTTP
headers/values that will override any headers associated
with the stored object in the response. See
http://goo.gl/06N3b for details.
:type version_id: string
:param version_id: Unused in this subclass.
:type generation: int
:param generation: A specific generation number to fetch the key at. If
not specified, the latest generation is fetched.
:rtype: :class:`boto.gs.key.Key`
:returns: A Key object from this bucket.
"""
query_args_l = []
if generation:
query_args_l.append('generation=%s' % generation)
if response_headers:
for rk, rv in six.iteritems(response_headers):
query_args_l.append('%s=%s' % (rk, urllib.quote(rv)))
try:
key, resp = self._get_key_internal(key_name,
headers,
query_args_l=query_args_l)
except GSResponseError as e:
if e.status == 403 and 'Forbidden' in e.reason:
# If we failed getting an object, let the user know which object
# failed rather than just returning a generic 403.
e.reason = ("Access denied to 'gs://%s/%s'." %
(self.name, key_name))
raise
return key
def build_put_params(self,
params,
name,
value=None,
timestamp=None,
unit=None,
dimensions=None,
statistics=None):
args = (name, value, unit, dimensions, statistics, timestamp)
length = max(map(lambda a: len(a) if isinstance(a, list) else 1, args))
def aslist(a):
if isinstance(a, list):
if len(a) != length:
raise Exception(
'Must specify equal number of elements; expected %d.' %
length)
return a
return [a] * length
for index, (n, v, u, d, s, t) in enumerate(zip(*map(aslist, args))):
metric_data = {'MetricName': n}
if timestamp:
metric_data['Timestamp'] = t.isoformat()
if unit:
metric_data['Unit'] = u
if dimensions:
self.build_dimension_param(d, metric_data)
if statistics:
metric_data['StatisticValues.Maximum'] = s['maximum']
metric_data['StatisticValues.Minimum'] = s['minimum']
metric_data['StatisticValues.SampleCount'] = s['samplecount']
metric_data['StatisticValues.Sum'] = s['sum']
if value is not None:
msg = 'You supplied a value and statistics for a ' + \
'metric.Posting statistics and not value.'
boto.log.warn(msg)
elif value is not None:
metric_data['Value'] = v
else:
raise Exception('Must specify a value or statistics to put.')
for key, val in six.iteritems(metric_data):
params['MetricData.member.%d.%s' % (index + 1, key)] = val
def create_lb_policy(self, lb_name, policy_name, policy_type,
policy_attributes):
"""
Creates a new policy that contains the necessary attributes
depending on the policy type. Policies are settings that are
saved for your load balancer and that can be applied to the
front-end listener, or the back-end application server.
"""
params = {'LoadBalancerName': lb_name,
'PolicyName': policy_name,
'PolicyTypeName': policy_type}
for index, (name, value) in enumerate(six.iteritems(policy_attributes), 1):
params['PolicyAttributes.member.%d.AttributeName' % index] = name
params['PolicyAttributes.member.%d.AttributeValue' % index] = value
else:
params['PolicyAttributes'] = ''
return self.get_status('CreateLoadBalancerPolicy', params)
def get_key(self, key_name, headers=None, version_id=None,
response_headers=None, generation=None):
"""Returns a Key instance for an object in this bucket.
Note that this method uses a HEAD request to check for the existence of
the key.
:type key_name: string
:param key_name: The name of the key to retrieve
:type response_headers: dict
:param response_headers: A dictionary containing HTTP
headers/values that will override any headers associated
with the stored object in the response. See
http://goo.gl/06N3b for details.
:type version_id: string
:param version_id: Unused in this subclass.
:type generation: int
:param generation: A specific generation number to fetch the key at. If
not specified, the latest generation is fetched.
:rtype: :class:`boto.gs.key.Key`
:returns: A Key object from this bucket.
"""
query_args_l = []
if generation:
query_args_l.append('generation=%s' % generation)
if response_headers:
for rk, rv in six.iteritems(response_headers):
query_args_l.append('%s=%s' % (rk, urllib.quote(rv)))
try:
key, resp = self._get_key_internal(key_name, headers,
query_args_l=query_args_l)
except GSResponseError as e:
if e.status == 403 and 'Forbidden' in e.reason:
# If we failed getting an object, let the user know which object
# failed rather than just returning a generic 403.
e.reason = ("Access denied to 'gs://%s/%s'." %
(self.name, key_name))
raise
return key
def build_put_params(self, params, name, value=None, timestamp=None,
unit=None, dimensions=None, statistics=None):
args = (name, value, unit, dimensions, statistics, timestamp)
length = max(map(lambda a: len(a) if isinstance(a, list) else 1, args))
def aslist(a):
if isinstance(a, list):
if len(a) != length:
raise Exception('Must specify equal number of elements; expected %d.' % length)
return a
return [a] * length
for index, (n, v, u, d, s, t) in enumerate(zip(*map(aslist, args))):
metric_data = {'MetricName': n}
if timestamp:
metric_data['Timestamp'] = t.isoformat()
if unit:
metric_data['Unit'] = u
if dimensions:
self.build_dimension_param(d, metric_data)
if statistics:
metric_data['StatisticValues.Maximum'] = s['maximum']
metric_data['StatisticValues.Minimum'] = s['minimum']
metric_data['StatisticValues.SampleCount'] = s['samplecount']
metric_data['StatisticValues.Sum'] = s['sum']
if value is not None:
msg = 'You supplied a value and statistics for a ' + \
'metric.Posting statistics and not value.'
boto.log.warn(msg)
elif value is not None:
metric_data['Value'] = v
else:
raise Exception('Must specify a value or statistics to put.')
for key, val in six.iteritems(metric_data):
params['MetricData.member.%d.%s' % (index + 1, key)] = val
def run_jobflow(self,
name,
log_uri=None,
ec2_keyname=None,
availability_zone=None,
main_instance_type='m1.small',
subordinate_instance_type='m1.small',
num_instances=1,
action_on_failure='TERMINATE_JOB_FLOW',
keep_alive=False,
enable_debugging=False,
hadoop_version=None,
steps=[],
bootstrap_actions=[],
instance_groups=None,
additional_info=None,
ami_version=None,
api_params=None,
visible_to_all_users=None,
job_flow_role=None,
service_role=None):
"""
Runs a job flow
:type name: str
:param name: Name of the job flow
:type log_uri: str
:param log_uri: URI of the S3 bucket to place logs
:type ec2_keyname: str
:param ec2_keyname: EC2 key used for the instances
:type availability_zone: str
:param availability_zone: EC2 availability zone of the cluster
:type main_instance_type: str
:param main_instance_type: EC2 instance type of the main
:type subordinate_instance_type: str
:param subordinate_instance_type: EC2 instance type of the subordinate nodes
:type num_instances: int
:param num_instances: Number of instances in the Hadoop cluster
:type action_on_failure: str
:param action_on_failure: Action to take if a step terminates
:type keep_alive: bool
:param keep_alive: Denotes whether the cluster should stay
alive upon completion
:type enable_debugging: bool
:param enable_debugging: Denotes whether AWS console debugging
should be enabled.
:type hadoop_version: str
:param hadoop_version: Version of Hadoop to use. This no longer
defaults to '0.20' and now uses the AMI default.
:type steps: list(boto.emr.Step)
:param steps: List of steps to add with the job
:type bootstrap_actions: list(boto.emr.BootstrapAction)
:param bootstrap_actions: List of bootstrap actions that run
before Hadoop starts.
:type instance_groups: list(boto.emr.InstanceGroup)
:param instance_groups: Optional list of instance groups to
use when creating this job.
NB: When provided, this argument supersedes num_instances
and main/subordinate_instance_type.
:type ami_version: str
:param ami_version: Amazon Machine Image (AMI) version to use
for instances. Values accepted by EMR are '1.0', '2.0', and
'latest'; EMR currently defaults to '1.0' if you don't set
'ami_version'.
:type additional_info: JSON str
:param additional_info: A JSON string for selecting additional features
:type api_params: dict
:param api_params: a dictionary of additional parameters to pass
directly to the EMR API (so you don't have to upgrade boto to
use new EMR features). You can also delete an API parameter
by setting it to None.
:type visible_to_all_users: bool
:param visible_to_all_users: Whether the job flow is visible to all IAM
users of the AWS account associated with the job flow. If this
value is set to ``True``, all IAM users of that AWS
account can view and (if they have the proper policy permissions
set) manage the job flow. If it is set to ``False``, only
the IAM user that created the job flow can view and manage
it.
:type job_flow_role: str
:param job_flow_role: An IAM role for the job flow. The EC2
instances of the job flow assume this role. The default role is
``EMRJobflowDefault``. In order to use the default role,
you must have already created it using the CLI.
:type service_role: str
:param service_role: The IAM role that will be assumed by the Amazon
EMR service to access AWS resources on your behalf.
:rtype: str
:return: The jobflow id
"""
params = {}
if action_on_failure:
params['ActionOnFailure'] = action_on_failure
if log_uri:
params['LogUri'] = log_uri
params['Name'] = name
# Common instance args
common_params = self._build_instance_common_args(
ec2_keyname, availability_zone, keep_alive, hadoop_version)
params.update(common_params)
# NB: according to the AWS API's error message, we must
# "configure instances either using instance count, main and
# subordinate instance type or instance groups but not both."
#
# Thus we switch here on the truthiness of instance_groups.
if not instance_groups:
# Instance args (the common case)
instance_params = self._build_instance_count_and_type_args(
main_instance_type, subordinate_instance_type, num_instances)
params.update(instance_params)
else:
# Instance group args (for spot instances or a heterogenous cluster)
list_args = self._build_instance_group_list_args(instance_groups)
instance_params = dict(
('Instances.%s' % k, v) for k, v in six.iteritems(list_args))
params.update(instance_params)
# Debugging step from EMR API docs
if enable_debugging:
debugging_step = JarStep(name='Setup Hadoop Debugging',
action_on_failure='TERMINATE_JOB_FLOW',
main_class=None,
jar=self.DebuggingJar,
step_args=self.DebuggingArgs)
steps.insert(0, debugging_step)
# Step args
if steps:
step_args = [self._build_step_args(step) for step in steps]
params.update(self._build_step_list(step_args))
if bootstrap_actions:
bootstrap_action_args = [
self._build_bootstrap_action_args(bootstrap_action)
for bootstrap_action in bootstrap_actions
]
params.update(
self._build_bootstrap_action_list(bootstrap_action_args))
if ami_version:
params['AmiVersion'] = ami_version
if additional_info is not None:
params['AdditionalInfo'] = additional_info
if api_params:
for key, value in six.iteritems(api_params):
if value is None:
params.pop(key, None)
else:
params[key] = value
if visible_to_all_users is not None:
if visible_to_all_users:
params['VisibleToAllUsers'] = 'true'
else:
params['VisibleToAllUsers'] = 'false'
if job_flow_role is not None:
params['JobFlowRole'] = job_flow_role
if service_role is not None:
params['ServiceRole'] = service_role
response = self.get_object('RunJobFlow',
params,
RunJobFlowResponse,
verb='POST')
return response.jobflowid
class MetricAlarm(object):
OK = 'OK'
ALARM = 'ALARM'
INSUFFICIENT_DATA = 'INSUFFICIENT_DATA'
_cmp_map = {
'>=': 'GreaterThanOrEqualToThreshold',
'>': 'GreaterThanThreshold',
'<': 'LessThanThreshold',
'<=': 'LessThanOrEqualToThreshold',
}
_rev_cmp_map = dict((v, k) for (k, v) in six.iteritems(_cmp_map))
def __init__(self, connection=None, name=None, metric=None,
namespace=None, statistic=None, comparison=None,
threshold=None, period=None, evaluation_periods=None,
unit=None, description='', dimensions=None,
alarm_actions=None, insufficient_data_actions=None,
ok_actions=None):
"""
Creates a new Alarm.
:type name: str
:param name: Name of alarm.
:type metric: str
:param metric: Name of alarm's associated metric.
:type namespace: str
:param namespace: The namespace for the alarm's metric.
:type statistic: str
:param statistic: The statistic to apply to the alarm's associated
metric.
Valid values: SampleCount|Average|Sum|Minimum|Maximum
:type comparison: str
:param comparison: Comparison used to compare statistic with threshold.
Valid values: >= | > | < | <=
:type threshold: float
:param threshold: The value against which the specified statistic
is compared.
:type period: int
:param period: The period in seconds over which teh specified
statistic is applied.
:type evaluation_periods: int
:param evaluation_periods: The number of periods over which data is
compared to the specified threshold.
:type unit: str
:param unit: Allowed Values are:
Seconds|Microseconds|Milliseconds,
Bytes|Kilobytes|Megabytes|Gigabytes|Terabytes,
Bits|Kilobits|Megabits|Gigabits|Terabits,
Percent|Count|
Bytes/Second|Kilobytes/Second|Megabytes/Second|
Gigabytes/Second|Terabytes/Second,
Bits/Second|Kilobits/Second|Megabits/Second,
Gigabits/Second|Terabits/Second|Count/Second|None
:type description: str
:param description: Description of MetricAlarm
:type dimensions: dict
:param dimensions: A dictionary of dimension key/values where
the key is the dimension name and the value
is either a scalar value or an iterator
of values to be associated with that
dimension.
Example: {
'InstanceId': ['i-0123456', 'i-0123457'],
'LoadBalancerName': 'test-lb'
}
:type alarm_actions: list of strs
:param alarm_actions: A list of the ARNs of the actions to take in
ALARM state
:type insufficient_data_actions: list of strs
:param insufficient_data_actions: A list of the ARNs of the actions to
take in INSUFFICIENT_DATA state
:type ok_actions: list of strs
:param ok_actions: A list of the ARNs of the actions to take in OK state
"""
self.name = name
self.connection = connection
self.metric = metric
self.namespace = namespace
self.statistic = statistic
if threshold is not None:
self.threshold = float(threshold)
else:
self.threshold = None
self.comparison = self._cmp_map.get(comparison)
if period is not None:
self.period = int(period)
else:
self.period = None
if evaluation_periods is not None:
self.evaluation_periods = int(evaluation_periods)
else:
self.evaluation_periods = None
self.actions_enabled = None
self.alarm_arn = None
self.last_updated = None
self.description = description
self.dimensions = dimensions
self.state_reason = None
self.state_value = None
self.unit = unit
self.alarm_actions = alarm_actions
self.insufficient_data_actions = insufficient_data_actions
self.ok_actions = ok_actions
def __repr__(self):
return 'MetricAlarm:%s[%s(%s) %s %s]' % (self.name, self.metric,
self.statistic,
self.comparison,
self.threshold)
def startElement(self, name, attrs, connection):
if name == 'AlarmActions':
self.alarm_actions = ListElement()
return self.alarm_actions
elif name == 'InsufficientDataActions':
self.insufficient_data_actions = ListElement()
return self.insufficient_data_actions
elif name == 'OKActions':
self.ok_actions = ListElement()
return self.ok_actions
elif name == 'Dimensions':
self.dimensions = Dimension()
return self.dimensions
else:
pass
def endElement(self, name, value, connection):
if name == 'ActionsEnabled':
self.actions_enabled = value
elif name == 'AlarmArn':
self.alarm_arn = value
elif name == 'AlarmConfigurationUpdatedTimestamp':
self.last_updated = value
elif name == 'AlarmDescription':
self.description = value
elif name == 'AlarmName':
self.name = value
elif name == 'ComparisonOperator':
setattr(self, 'comparison', self._rev_cmp_map[value])
elif name == 'EvaluationPeriods':
self.evaluation_periods = int(value)
elif name == 'MetricName':
self.metric = value
elif name == 'Namespace':
self.namespace = value
elif name == 'Period':
self.period = int(value)
elif name == 'StateReason':
self.state_reason = value
elif name == 'StateValue':
self.state_value = value
elif name == 'Statistic':
self.statistic = value
elif name == 'Threshold':
self.threshold = float(value)
elif name == 'Unit':
self.unit = value
else:
setattr(self, name, value)
def set_state(self, value, reason, data=None):
""" Temporarily sets the state of an alarm.
:type value: str
:param value: OK | ALARM | INSUFFICIENT_DATA
:type reason: str
:param reason: Reason alarm set (human readable).
:type data: str
:param data: Reason data (will be jsonified).
"""
return self.connection.set_alarm_state(self.name, reason, value, data)
def update(self):
return self.connection.update_alarm(self)
def enable_actions(self):
return self.connection.enable_alarm_actions([self.name])
def disable_actions(self):
return self.connection.disable_alarm_actions([self.name])
def describe_history(self, start_date=None, end_date=None, max_records=None,
history_item_type=None, next_token=None):
return self.connection.describe_alarm_history(self.name, start_date,
end_date, max_records,
history_item_type,
next_token)
def add_alarm_action(self, action_arn=None):
"""
Adds an alarm action, represented as an SNS topic, to this alarm.
What do do when alarm is triggered.
:type action_arn: str
:param action_arn: SNS topics to which notification should be
sent if the alarm goes to state ALARM.
"""
if not action_arn:
return # Raise exception instead?
self.actions_enabled = 'true'
self.alarm_actions.append(action_arn)
def add_insufficient_data_action(self, action_arn=None):
"""
Adds an insufficient_data action, represented as an SNS topic, to
this alarm. What to do when the insufficient_data state is reached.
:type action_arn: str
:param action_arn: SNS topics to which notification should be
sent if the alarm goes to state INSUFFICIENT_DATA.
"""
if not action_arn:
return
self.actions_enabled = 'true'
self.insufficient_data_actions.append(action_arn)
def add_ok_action(self, action_arn=None):
"""
Adds an ok action, represented as an SNS topic, to this alarm. What
to do when the ok state is reached.
:type action_arn: str
:param action_arn: SNS topics to which notification should be
sent if the alarm goes to state INSUFFICIENT_DATA.
"""
if not action_arn:
return
self.actions_enabled = 'true'
self.ok_actions.append(action_arn)
def delete(self):
self.connection.delete_alarms([self.name])
def get(elb, name):
"""Get details about ELB <name>"""
b = find_elb(elb, name)
if b:
print "=" * 80
print "Name: %s" % b.name
print "DNS Name: %s" % b.dns_name
if b.canonical_hosted_zone_name:
chzn = b.canonical_hosted_zone_name
print "Canonical hosted zone name: %s" % chzn
if b.canonical_hosted_zone_name_id:
chznid = b.canonical_hosted_zone_name_id
print "Canonical hosted zone name id: %s" % chznid
print
print "Health Check: %s" % b.health_check
print
print "Listeners"
print "---------"
print "%-8s %-8s %s" % ("IN", "OUT", "PROTO")
for l in b.listeners:
print "%-8s %-8s %s" % (l[0], l[1], l[2])
print
print " Zones "
print "---------"
for z in b.availability_zones:
print z
print
# Make map of all instance Id's to Name tags
import boto
from boto.compat.six import iteritems
if not options.region:
ec2 = boto.connect_ec2()
else:
ec2 = boto.ec2.connect_to_region(options.region)
check_valid_region(ec2, options.region)
instance_health = b.get_instance_health()
instances = [state.instance_id for state in instance_health]
names = dict((k,'') for k in instances)
for i in ec2.get_only_instances():
if i.id in instances:
names[i.id] = i.tags.get('Name', '')
name_column_width = max([4] + [len(v) for k,v in iteritems(names)]) + 2
print "Instances"
print "---------"
print "%-12s %-15s %-*s %s" % ("ID",
"STATE",
name_column_width, "NAME",
"DESCRIPTION")
for state in instance_health:
print "%-12s %-15s %-*s %s" % (state.instance_id,
state.state,
name_column_width, names[state.instance_id],
state.description)
print
def run_jobflow(self, name, log_uri=None, ec2_keyname=None,
availability_zone=None,
master_instance_type='m1.small',
slave_instance_type='m1.small', num_instances=1,
action_on_failure='TERMINATE_JOB_FLOW', keep_alive=False,
enable_debugging=False,
hadoop_version=None,
steps=[],
bootstrap_actions=[],
instance_groups=None,
additional_info=None,
ami_version=None,
api_params=None,
visible_to_all_users=None,
job_flow_role=None,
service_role=None):
"""
Runs a job flow
:type name: str
:param name: Name of the job flow
:type log_uri: str
:param log_uri: URI of the S3 bucket to place logs
:type ec2_keyname: str
:param ec2_keyname: EC2 key used for the instances
:type availability_zone: str
:param availability_zone: EC2 availability zone of the cluster
:type master_instance_type: str
:param master_instance_type: EC2 instance type of the master
:type slave_instance_type: str
:param slave_instance_type: EC2 instance type of the slave nodes
:type num_instances: int
:param num_instances: Number of instances in the Hadoop cluster
:type action_on_failure: str
:param action_on_failure: Action to take if a step terminates
:type keep_alive: bool
:param keep_alive: Denotes whether the cluster should stay
alive upon completion
:type enable_debugging: bool
:param enable_debugging: Denotes whether AWS console debugging
should be enabled.
:type hadoop_version: str
:param hadoop_version: Version of Hadoop to use. This no longer
defaults to '0.20' and now uses the AMI default.
:type steps: list(boto.emr.Step)
:param steps: List of steps to add with the job
:type bootstrap_actions: list(boto.emr.BootstrapAction)
:param bootstrap_actions: List of bootstrap actions that run
before Hadoop starts.
:type instance_groups: list(boto.emr.InstanceGroup)
:param instance_groups: Optional list of instance groups to
use when creating this job.
NB: When provided, this argument supersedes num_instances
and master/slave_instance_type.
:type ami_version: str
:param ami_version: Amazon Machine Image (AMI) version to use
for instances. Values accepted by EMR are '1.0', '2.0', and
'latest'; EMR currently defaults to '1.0' if you don't set
'ami_version'.
:type additional_info: JSON str
:param additional_info: A JSON string for selecting additional features
:type api_params: dict
:param api_params: a dictionary of additional parameters to pass
directly to the EMR API (so you don't have to upgrade boto to
use new EMR features). You can also delete an API parameter
by setting it to None.
:type visible_to_all_users: bool
:param visible_to_all_users: Whether the job flow is visible to all IAM
users of the AWS account associated with the job flow. If this
value is set to ``True``, all IAM users of that AWS
account can view and (if they have the proper policy permissions
set) manage the job flow. If it is set to ``False``, only
the IAM user that created the job flow can view and manage
it.
:type job_flow_role: str
:param job_flow_role: An IAM role for the job flow. The EC2
instances of the job flow assume this role. The default role is
``EMRJobflowDefault``. In order to use the default role,
you must have already created it using the CLI.
:type service_role: str
:param service_role: The IAM role that will be assumed by the Amazon
EMR service to access AWS resources on your behalf.
:rtype: str
:return: The jobflow id
"""
params = {}
if action_on_failure:
params['ActionOnFailure'] = action_on_failure
if log_uri:
params['LogUri'] = log_uri
params['Name'] = name
# Common instance args
common_params = self._build_instance_common_args(ec2_keyname,
availability_zone,
keep_alive,
hadoop_version)
params.update(common_params)
# NB: according to the AWS API's error message, we must
# "configure instances either using instance count, master and
# slave instance type or instance groups but not both."
#
# Thus we switch here on the truthiness of instance_groups.
if not instance_groups:
# Instance args (the common case)
instance_params = self._build_instance_count_and_type_args(
master_instance_type,
slave_instance_type,
num_instances)
params.update(instance_params)
else:
# Instance group args (for spot instances or a heterogenous cluster)
list_args = self._build_instance_group_list_args(instance_groups)
instance_params = dict(
('Instances.%s' % k, v) for k, v in six.iteritems(list_args)
)
params.update(instance_params)
# Debugging step from EMR API docs
if enable_debugging:
debugging_step = JarStep(name='Setup Hadoop Debugging',
action_on_failure='TERMINATE_JOB_FLOW',
main_class=None,
jar=self.DebuggingJar,
step_args=self.DebuggingArgs)
steps.insert(0, debugging_step)
# Step args
if steps:
step_args = [self._build_step_args(step) for step in steps]
params.update(self._build_step_list(step_args))
if bootstrap_actions:
bootstrap_action_args = [self._build_bootstrap_action_args(bootstrap_action) for bootstrap_action in bootstrap_actions]
params.update(self._build_bootstrap_action_list(bootstrap_action_args))
if ami_version:
params['AmiVersion'] = ami_version
if additional_info is not None:
params['AdditionalInfo'] = additional_info
if api_params:
for key, value in six.iteritems(api_params):
if value is None:
params.pop(key, None)
else:
params[key] = value
if visible_to_all_users is not None:
if visible_to_all_users:
params['VisibleToAllUsers'] = 'true'
else:
params['VisibleToAllUsers'] = 'false'
if job_flow_role is not None:
params['JobFlowRole'] = job_flow_role
if service_role is not None:
params['ServiceRole'] = service_role
response = self.get_object(
'RunJobFlow', params, RunJobFlowResponse, verb='POST')
return response.jobflowid
