def build_list_params(self, params, items, label): """ Items is a list of dictionaries or strings:: [ { 'Protocol' : 'HTTP', 'LoadBalancerPort' : '80', 'InstancePort' : '80' }, .. ] etc. or:: ['us-east-1b',...] """ # different from EC2 list params for i in range(1, len(items) + 1): if isinstance(items[i - 1], dict): for k, v in six.iteritems(items[i - 1]): if isinstance(v, dict): for kk, vv in six.iteritems(v): params['%s.member.%d.%s.%s' % (label, i, k, kk)] = vv else: params['%s.member.%d.%s' % (label, i, k)] = v elif isinstance(items[i - 1], six.string_types): params['%s.member.%d' % (label, i)] = items[i - 1]
def to_domain_connection_params(self): """ Transform search parameters from instance properties to a dictionary that CloudSearchDomainConnection can accept :rtype: dict :return: search parameters """ params = {'start': self.start, 'size': self.real_size} if self.q: params['q'] = self.q if self.parser: params['query_parser'] = self.parser if self.fq: params['filter_query'] = self.fq if self.expr: expr = {} for k, v in six.iteritems(self.expr): expr['expr.%s' % k] = v params['expr'] = expr if self.facet: facet = {} for k, v in six.iteritems(self.facet): if not isinstance(v, six.string_types): v = json.dumps(v) facet['facet.%s' % k] = v params['facet'] = facet if self.highlight: highlight = {} for k, v in six.iteritems(self.highlight): highlight['highlight.%s' % k] = v params['highlight'] = highlight if self.options: params['query_options'] = self.options if self.return_fields: params['ret'] = ','.join(self.return_fields) if self.partial is not None: params['partial'] = self.partial if self.sort: params['sort'] = ','.join(self.sort) return params
def resubmit(self): """ Resubmit the batch to get the next result set. The request object is rebuild from scratch meaning that all batch added between ``submit`` and ``resubmit`` will be lost. Note: This method is experimental and subject to changes in future releases """ del self[:] if not self.unprocessed: return None for table_name, table_req in six.iteritems(self.unprocessed): table_keys = table_req['Keys'] table = self.layer2.get(table_name) keys = [] for key in table_keys: h = key['HashKeyElement'] r = None if 'RangeKeyElement' in key: r = key['RangeKeyElement'] keys.append((h, r)) attributes_to_get = None if 'AttributesToGet' in table_req: attributes_to_get = table_req['AttributesToGet'] self.add_batch(table, keys, attributes_to_get=attributes_to_get) return self.submit()
def _build_step_list(self, steps): if not isinstance(steps, list): steps = [steps] params = {} for i, step in enumerate(steps): for key, value in six.iteritems(step): params['Steps.member.%s.%s' % (i+1, key)] = value return params
def _build_bootstrap_action_list(self, bootstrap_actions): if not isinstance(bootstrap_actions, list): bootstrap_actions = [bootstrap_actions] params = {} for i, bootstrap_action in enumerate(bootstrap_actions): for key, value in six.iteritems(bootstrap_action): params['BootstrapActions.member.%s.%s' % (i + 1, key)] = value return params
def to_params(self): """Transform search parameters from instance properties to a dictionary :rtype: dict :return: search parameters """ params = {"start": self.start, "size": self.real_size} if self.q: params["q"] = self.q if self.parser: params["q.parser"] = self.parser if self.fq: params["fq"] = self.fq if self.expr: for k, v in six.iteritems(self.expr): params["expr.%s" % k] = v if self.facet: for k, v in six.iteritems(self.facet): if type(v) not in [str, unicode]: v = json.dumps(v) params["facet.%s" % k] = v if self.highlight: for k, v in six.iteritems(self.highlight): params["highlight.%s" % k] = v if self.options: params["options"] = self.options if self.return_fields: params["return"] = ",".join(self.return_fields) if self.partial is not None: params["partial"] = self.partial if self.sort: params["sort"] = ",".join(self.sort) return params
def to_params(self): """Transform search parameters from instance properties to a dictionary :rtype: dict :return: search parameters """ params = {'start': self.start, 'size': self.real_size} if self.q: params['q'] = self.q if self.parser: params['q.parser'] = self.parser if self.fq: params['fq'] = self.fq if self.expr: for k, v in six.iteritems(self.expr): params['expr.%s' % k] = v if self.facet: for k, v in six.iteritems(self.facet): if not isinstance(v, six.string_types): v = json.dumps(v) params['facet.%s' % k] = v if self.highlight: for k, v in six.iteritems(self.highlight): params['highlight.%s' % k] = v if self.options: params['options'] = self.options if self.return_fields: params['return'] = ','.join(self.return_fields) if self.partial is not None: params['partial'] = self.partial if self.sort: params['sort'] = ','.join(self.sort) return params
def make_request(self, action, path, headers=None, data='', params=None): if params: pairs = [] for key, val in six.iteritems(params): if val is None: continue pairs.append(key + '=' + urllib.parse.quote(str(val))) path += '?' + '&'.join(pairs) return super(Route53Connection, self).make_request( action, path, headers, data, retry_handler=self._retry_handler)
def to_params(self): """Transform search parameters from instance properties to a dictionary :rtype: dict :return: search parameters """ params = {'start': self.start, 'size': self.real_size} if self.q: params['q'] = self.q if self.bq: params['bq'] = self.bq if self.rank_expressions: for k, v in self.rank_expressions.iteritems(): params['rank-%s' % k] = v if self.rank: params['rank'] = ','.join(self.rank) if self.return_fields: params['return-fields'] = ','.join(self.return_fields) if self.facet: params['facet'] = ','.join(self.facet) if self.facet_constraints: for k, v in six.iteritems(self.facet_constraints): params['facet-%s-constraints' % k] = v if self.facet_sort: for k, v in six.iteritems(self.facet_sort): params['facet-%s-sort' % k] = v if self.facet_top_n: for k, v in six.iteritems(self.facet_top_n): params['facet-%s-top-n' % k] = v if self.t: for k, v in six.iteritems(self.t): params['t-%s' % k] = v return params
def _build_tag_list(self, tags): assert isinstance(tags, dict) params = {} for i, key_value in enumerate(sorted(six.iteritems(tags)), start=1): key, value = key_value current_prefix = 'Tags.member.%s' % i params['%s.Key' % current_prefix] = key if value: params['%s.Value' % current_prefix] = value return params
def build_list_params(self, params, items, label): if isinstance(items, six.string_types): items = [items] for index, item in enumerate(items): i = index + 1 if isinstance(item, dict): for k, v in six.iteritems(item): params[label % (i, 'Name')] = k if v is not None: params[label % (i, 'Value')] = v else: params[label % i] = item
def __repr__(self): result = self.__class__.__name__ + '{ ' counter = 0 for key, value in six.iteritems(self.__dict__): # first iteration no comma counter += 1 if counter > 1: result += ', ' result += key + ': ' result += self._repr_by_type(value) result += ' }' return result
def create_lb_policy(self, lb_name, policy_name, policy_type, policy_attributes): """ Creates a new policy that contains the necessary attributes depending on the policy type. Policies are settings that are saved for your load balancer and that can be applied to the front-end listener, or the back-end application server. """ params = {"LoadBalancerName": lb_name, "PolicyName": policy_name, "PolicyTypeName": policy_type} for index, (name, value) in enumerate(six.iteritems(policy_attributes), 1): params["PolicyAttributes.member.%d.AttributeName" % index] = name params["PolicyAttributes.member.%d.AttributeValue" % index] = value else: params["PolicyAttributes"] = "" return self.get_status("CreateLoadBalancerPolicy", params)
def _build_instance_group_list_args(self, instance_groups): """ Takes a list of InstanceGroups, or a single InstanceGroup. Returns a comparable dict for use in making a RunJobFlow or AddInstanceGroups request. """ if not isinstance(instance_groups, list): instance_groups = [instance_groups] params = {} for i, instance_group in enumerate(instance_groups): ig_dict = self._build_instance_group_args(instance_group) for key, value in six.iteritems(ig_dict): params['InstanceGroups.member.%d.%s' % (i + 1, key)] = value return params
def _build_instance_group_list_args(self, instance_groups): """ Takes a list of InstanceGroups, or a single InstanceGroup. Returns a comparable dict for use in making a RunJobFlow or AddInstanceGroups request. """ if not isinstance(instance_groups, list): instance_groups = [instance_groups] params = {} for i, instance_group in enumerate(instance_groups): ig_dict = self._build_instance_group_args(instance_group) for key, value in six.iteritems(ig_dict): params['InstanceGroups.member.%d.%s' % (i+1, key)] = value return params
def get_key(self, key_name, headers=None, version_id=None, response_headers=None, generation=None): """Returns a Key instance for an object in this bucket. Note that this method uses a HEAD request to check for the existence of the key. :type key_name: string :param key_name: The name of the key to retrieve :type response_headers: dict :param response_headers: A dictionary containing HTTP headers/values that will override any headers associated with the stored object in the response. See http://goo.gl/06N3b for details. :type version_id: string :param version_id: Unused in this subclass. :type generation: int :param generation: A specific generation number to fetch the key at. If not specified, the latest generation is fetched. :rtype: :class:`boto.gs.key.Key` :returns: A Key object from this bucket. """ query_args_l = [] if generation: query_args_l.append('generation=%s' % generation) if response_headers: for rk, rv in six.iteritems(response_headers): query_args_l.append('%s=%s' % (rk, urllib.quote(rv))) try: key, resp = self._get_key_internal(key_name, headers, query_args_l=query_args_l) except GSResponseError as e: if e.status == 403 and 'Forbidden' in e.reason: # If we failed getting an object, let the user know which object # failed rather than just returning a generic 403. e.reason = ("Access denied to 'gs://%s/%s'." % (self.name, key_name)) raise return key
def build_put_params(self, params, name, value=None, timestamp=None, unit=None, dimensions=None, statistics=None): args = (name, value, unit, dimensions, statistics, timestamp) length = max(map(lambda a: len(a) if isinstance(a, list) else 1, args)) def aslist(a): if isinstance(a, list): if len(a) != length: raise Exception( 'Must specify equal number of elements; expected %d.' % length) return a return [a] * length for index, (n, v, u, d, s, t) in enumerate(zip(*map(aslist, args))): metric_data = {'MetricName': n} if timestamp: metric_data['Timestamp'] = t.isoformat() if unit: metric_data['Unit'] = u if dimensions: self.build_dimension_param(d, metric_data) if statistics: metric_data['StatisticValues.Maximum'] = s['maximum'] metric_data['StatisticValues.Minimum'] = s['minimum'] metric_data['StatisticValues.SampleCount'] = s['samplecount'] metric_data['StatisticValues.Sum'] = s['sum'] if value is not None: msg = 'You supplied a value and statistics for a ' + \ 'metric.Posting statistics and not value.' boto.log.warn(msg) elif value is not None: metric_data['Value'] = v else: raise Exception('Must specify a value or statistics to put.') for key, val in six.iteritems(metric_data): params['MetricData.member.%d.%s' % (index + 1, key)] = val
def create_lb_policy(self, lb_name, policy_name, policy_type, policy_attributes): """ Creates a new policy that contains the necessary attributes depending on the policy type. Policies are settings that are saved for your load balancer and that can be applied to the front-end listener, or the back-end application server. """ params = {'LoadBalancerName': lb_name, 'PolicyName': policy_name, 'PolicyTypeName': policy_type} for index, (name, value) in enumerate(six.iteritems(policy_attributes), 1): params['PolicyAttributes.member.%d.AttributeName' % index] = name params['PolicyAttributes.member.%d.AttributeValue' % index] = value else: params['PolicyAttributes'] = '' return self.get_status('CreateLoadBalancerPolicy', params)
def build_put_params(self, params, name, value=None, timestamp=None, unit=None, dimensions=None, statistics=None): args = (name, value, unit, dimensions, statistics, timestamp) length = max(map(lambda a: len(a) if isinstance(a, list) else 1, args)) def aslist(a): if isinstance(a, list): if len(a) != length: raise Exception('Must specify equal number of elements; expected %d.' % length) return a return [a] * length for index, (n, v, u, d, s, t) in enumerate(zip(*map(aslist, args))): metric_data = {'MetricName': n} if timestamp: metric_data['Timestamp'] = t.isoformat() if unit: metric_data['Unit'] = u if dimensions: self.build_dimension_param(d, metric_data) if statistics: metric_data['StatisticValues.Maximum'] = s['maximum'] metric_data['StatisticValues.Minimum'] = s['minimum'] metric_data['StatisticValues.SampleCount'] = s['samplecount'] metric_data['StatisticValues.Sum'] = s['sum'] if value is not None: msg = 'You supplied a value and statistics for a ' + \ 'metric.Posting statistics and not value.' boto.log.warn(msg) elif value is not None: metric_data['Value'] = v else: raise Exception('Must specify a value or statistics to put.') for key, val in six.iteritems(metric_data): params['MetricData.member.%d.%s' % (index + 1, key)] = val
def run_jobflow(self, name, log_uri=None, ec2_keyname=None, availability_zone=None, main_instance_type='m1.small', subordinate_instance_type='m1.small', num_instances=1, action_on_failure='TERMINATE_JOB_FLOW', keep_alive=False, enable_debugging=False, hadoop_version=None, steps=[], bootstrap_actions=[], instance_groups=None, additional_info=None, ami_version=None, api_params=None, visible_to_all_users=None, job_flow_role=None, service_role=None): """ Runs a job flow :type name: str :param name: Name of the job flow :type log_uri: str :param log_uri: URI of the S3 bucket to place logs :type ec2_keyname: str :param ec2_keyname: EC2 key used for the instances :type availability_zone: str :param availability_zone: EC2 availability zone of the cluster :type main_instance_type: str :param main_instance_type: EC2 instance type of the main :type subordinate_instance_type: str :param subordinate_instance_type: EC2 instance type of the subordinate nodes :type num_instances: int :param num_instances: Number of instances in the Hadoop cluster :type action_on_failure: str :param action_on_failure: Action to take if a step terminates :type keep_alive: bool :param keep_alive: Denotes whether the cluster should stay alive upon completion :type enable_debugging: bool :param enable_debugging: Denotes whether AWS console debugging should be enabled. :type hadoop_version: str :param hadoop_version: Version of Hadoop to use. This no longer defaults to '0.20' and now uses the AMI default. :type steps: list(boto.emr.Step) :param steps: List of steps to add with the job :type bootstrap_actions: list(boto.emr.BootstrapAction) :param bootstrap_actions: List of bootstrap actions that run before Hadoop starts. :type instance_groups: list(boto.emr.InstanceGroup) :param instance_groups: Optional list of instance groups to use when creating this job. NB: When provided, this argument supersedes num_instances and main/subordinate_instance_type. :type ami_version: str :param ami_version: Amazon Machine Image (AMI) version to use for instances. Values accepted by EMR are '1.0', '2.0', and 'latest'; EMR currently defaults to '1.0' if you don't set 'ami_version'. :type additional_info: JSON str :param additional_info: A JSON string for selecting additional features :type api_params: dict :param api_params: a dictionary of additional parameters to pass directly to the EMR API (so you don't have to upgrade boto to use new EMR features). You can also delete an API parameter by setting it to None. :type visible_to_all_users: bool :param visible_to_all_users: Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. If this value is set to ``True``, all IAM users of that AWS account can view and (if they have the proper policy permissions set) manage the job flow. If it is set to ``False``, only the IAM user that created the job flow can view and manage it. :type job_flow_role: str :param job_flow_role: An IAM role for the job flow. The EC2 instances of the job flow assume this role. The default role is ``EMRJobflowDefault``. In order to use the default role, you must have already created it using the CLI. :type service_role: str :param service_role: The IAM role that will be assumed by the Amazon EMR service to access AWS resources on your behalf. :rtype: str :return: The jobflow id """ params = {} if action_on_failure: params['ActionOnFailure'] = action_on_failure if log_uri: params['LogUri'] = log_uri params['Name'] = name # Common instance args common_params = self._build_instance_common_args( ec2_keyname, availability_zone, keep_alive, hadoop_version) params.update(common_params) # NB: according to the AWS API's error message, we must # "configure instances either using instance count, main and # subordinate instance type or instance groups but not both." # # Thus we switch here on the truthiness of instance_groups. if not instance_groups: # Instance args (the common case) instance_params = self._build_instance_count_and_type_args( main_instance_type, subordinate_instance_type, num_instances) params.update(instance_params) else: # Instance group args (for spot instances or a heterogenous cluster) list_args = self._build_instance_group_list_args(instance_groups) instance_params = dict( ('Instances.%s' % k, v) for k, v in six.iteritems(list_args)) params.update(instance_params) # Debugging step from EMR API docs if enable_debugging: debugging_step = JarStep(name='Setup Hadoop Debugging', action_on_failure='TERMINATE_JOB_FLOW', main_class=None, jar=self.DebuggingJar, step_args=self.DebuggingArgs) steps.insert(0, debugging_step) # Step args if steps: step_args = [self._build_step_args(step) for step in steps] params.update(self._build_step_list(step_args)) if bootstrap_actions: bootstrap_action_args = [ self._build_bootstrap_action_args(bootstrap_action) for bootstrap_action in bootstrap_actions ] params.update( self._build_bootstrap_action_list(bootstrap_action_args)) if ami_version: params['AmiVersion'] = ami_version if additional_info is not None: params['AdditionalInfo'] = additional_info if api_params: for key, value in six.iteritems(api_params): if value is None: params.pop(key, None) else: params[key] = value if visible_to_all_users is not None: if visible_to_all_users: params['VisibleToAllUsers'] = 'true' else: params['VisibleToAllUsers'] = 'false' if job_flow_role is not None: params['JobFlowRole'] = job_flow_role if service_role is not None: params['ServiceRole'] = service_role response = self.get_object('RunJobFlow', params, RunJobFlowResponse, verb='POST') return response.jobflowid
class MetricAlarm(object): OK = 'OK' ALARM = 'ALARM' INSUFFICIENT_DATA = 'INSUFFICIENT_DATA' _cmp_map = { '>=': 'GreaterThanOrEqualToThreshold', '>': 'GreaterThanThreshold', '<': 'LessThanThreshold', '<=': 'LessThanOrEqualToThreshold', } _rev_cmp_map = dict((v, k) for (k, v) in six.iteritems(_cmp_map)) def __init__(self, connection=None, name=None, metric=None, namespace=None, statistic=None, comparison=None, threshold=None, period=None, evaluation_periods=None, unit=None, description='', dimensions=None, alarm_actions=None, insufficient_data_actions=None, ok_actions=None): """ Creates a new Alarm. :type name: str :param name: Name of alarm. :type metric: str :param metric: Name of alarm's associated metric. :type namespace: str :param namespace: The namespace for the alarm's metric. :type statistic: str :param statistic: The statistic to apply to the alarm's associated metric. Valid values: SampleCount|Average|Sum|Minimum|Maximum :type comparison: str :param comparison: Comparison used to compare statistic with threshold. Valid values: >= | > | < | <= :type threshold: float :param threshold: The value against which the specified statistic is compared. :type period: int :param period: The period in seconds over which teh specified statistic is applied. :type evaluation_periods: int :param evaluation_periods: The number of periods over which data is compared to the specified threshold. :type unit: str :param unit: Allowed Values are: Seconds|Microseconds|Milliseconds, Bytes|Kilobytes|Megabytes|Gigabytes|Terabytes, Bits|Kilobits|Megabits|Gigabits|Terabits, Percent|Count| Bytes/Second|Kilobytes/Second|Megabytes/Second| Gigabytes/Second|Terabytes/Second, Bits/Second|Kilobits/Second|Megabits/Second, Gigabits/Second|Terabits/Second|Count/Second|None :type description: str :param description: Description of MetricAlarm :type dimensions: dict :param dimensions: A dictionary of dimension key/values where the key is the dimension name and the value is either a scalar value or an iterator of values to be associated with that dimension. Example: { 'InstanceId': ['i-0123456', 'i-0123457'], 'LoadBalancerName': 'test-lb' } :type alarm_actions: list of strs :param alarm_actions: A list of the ARNs of the actions to take in ALARM state :type insufficient_data_actions: list of strs :param insufficient_data_actions: A list of the ARNs of the actions to take in INSUFFICIENT_DATA state :type ok_actions: list of strs :param ok_actions: A list of the ARNs of the actions to take in OK state """ self.name = name self.connection = connection self.metric = metric self.namespace = namespace self.statistic = statistic if threshold is not None: self.threshold = float(threshold) else: self.threshold = None self.comparison = self._cmp_map.get(comparison) if period is not None: self.period = int(period) else: self.period = None if evaluation_periods is not None: self.evaluation_periods = int(evaluation_periods) else: self.evaluation_periods = None self.actions_enabled = None self.alarm_arn = None self.last_updated = None self.description = description self.dimensions = dimensions self.state_reason = None self.state_value = None self.unit = unit self.alarm_actions = alarm_actions self.insufficient_data_actions = insufficient_data_actions self.ok_actions = ok_actions def __repr__(self): return 'MetricAlarm:%s[%s(%s) %s %s]' % (self.name, self.metric, self.statistic, self.comparison, self.threshold) def startElement(self, name, attrs, connection): if name == 'AlarmActions': self.alarm_actions = ListElement() return self.alarm_actions elif name == 'InsufficientDataActions': self.insufficient_data_actions = ListElement() return self.insufficient_data_actions elif name == 'OKActions': self.ok_actions = ListElement() return self.ok_actions elif name == 'Dimensions': self.dimensions = Dimension() return self.dimensions else: pass def endElement(self, name, value, connection): if name == 'ActionsEnabled': self.actions_enabled = value elif name == 'AlarmArn': self.alarm_arn = value elif name == 'AlarmConfigurationUpdatedTimestamp': self.last_updated = value elif name == 'AlarmDescription': self.description = value elif name == 'AlarmName': self.name = value elif name == 'ComparisonOperator': setattr(self, 'comparison', self._rev_cmp_map[value]) elif name == 'EvaluationPeriods': self.evaluation_periods = int(value) elif name == 'MetricName': self.metric = value elif name == 'Namespace': self.namespace = value elif name == 'Period': self.period = int(value) elif name == 'StateReason': self.state_reason = value elif name == 'StateValue': self.state_value = value elif name == 'Statistic': self.statistic = value elif name == 'Threshold': self.threshold = float(value) elif name == 'Unit': self.unit = value else: setattr(self, name, value) def set_state(self, value, reason, data=None): """ Temporarily sets the state of an alarm. :type value: str :param value: OK | ALARM | INSUFFICIENT_DATA :type reason: str :param reason: Reason alarm set (human readable). :type data: str :param data: Reason data (will be jsonified). """ return self.connection.set_alarm_state(self.name, reason, value, data) def update(self): return self.connection.update_alarm(self) def enable_actions(self): return self.connection.enable_alarm_actions([self.name]) def disable_actions(self): return self.connection.disable_alarm_actions([self.name]) def describe_history(self, start_date=None, end_date=None, max_records=None, history_item_type=None, next_token=None): return self.connection.describe_alarm_history(self.name, start_date, end_date, max_records, history_item_type, next_token) def add_alarm_action(self, action_arn=None): """ Adds an alarm action, represented as an SNS topic, to this alarm. What do do when alarm is triggered. :type action_arn: str :param action_arn: SNS topics to which notification should be sent if the alarm goes to state ALARM. """ if not action_arn: return # Raise exception instead? self.actions_enabled = 'true' self.alarm_actions.append(action_arn) def add_insufficient_data_action(self, action_arn=None): """ Adds an insufficient_data action, represented as an SNS topic, to this alarm. What to do when the insufficient_data state is reached. :type action_arn: str :param action_arn: SNS topics to which notification should be sent if the alarm goes to state INSUFFICIENT_DATA. """ if not action_arn: return self.actions_enabled = 'true' self.insufficient_data_actions.append(action_arn) def add_ok_action(self, action_arn=None): """ Adds an ok action, represented as an SNS topic, to this alarm. What to do when the ok state is reached. :type action_arn: str :param action_arn: SNS topics to which notification should be sent if the alarm goes to state INSUFFICIENT_DATA. """ if not action_arn: return self.actions_enabled = 'true' self.ok_actions.append(action_arn) def delete(self): self.connection.delete_alarms([self.name])
def get(elb, name): """Get details about ELB <name>""" b = find_elb(elb, name) if b: print "=" * 80 print "Name: %s" % b.name print "DNS Name: %s" % b.dns_name if b.canonical_hosted_zone_name: chzn = b.canonical_hosted_zone_name print "Canonical hosted zone name: %s" % chzn if b.canonical_hosted_zone_name_id: chznid = b.canonical_hosted_zone_name_id print "Canonical hosted zone name id: %s" % chznid print print "Health Check: %s" % b.health_check print print "Listeners" print "---------" print "%-8s %-8s %s" % ("IN", "OUT", "PROTO") for l in b.listeners: print "%-8s %-8s %s" % (l[0], l[1], l[2]) print print " Zones " print "---------" for z in b.availability_zones: print z print # Make map of all instance Id's to Name tags import boto from boto.compat.six import iteritems if not options.region: ec2 = boto.connect_ec2() else: ec2 = boto.ec2.connect_to_region(options.region) check_valid_region(ec2, options.region) instance_health = b.get_instance_health() instances = [state.instance_id for state in instance_health] names = dict((k,'') for k in instances) for i in ec2.get_only_instances(): if i.id in instances: names[i.id] = i.tags.get('Name', '') name_column_width = max([4] + [len(v) for k,v in iteritems(names)]) + 2 print "Instances" print "---------" print "%-12s %-15s %-*s %s" % ("ID", "STATE", name_column_width, "NAME", "DESCRIPTION") for state in instance_health: print "%-12s %-15s %-*s %s" % (state.instance_id, state.state, name_column_width, names[state.instance_id], state.description) print
def run_jobflow(self, name, log_uri=None, ec2_keyname=None, availability_zone=None, master_instance_type='m1.small', slave_instance_type='m1.small', num_instances=1, action_on_failure='TERMINATE_JOB_FLOW', keep_alive=False, enable_debugging=False, hadoop_version=None, steps=[], bootstrap_actions=[], instance_groups=None, additional_info=None, ami_version=None, api_params=None, visible_to_all_users=None, job_flow_role=None, service_role=None): """ Runs a job flow :type name: str :param name: Name of the job flow :type log_uri: str :param log_uri: URI of the S3 bucket to place logs :type ec2_keyname: str :param ec2_keyname: EC2 key used for the instances :type availability_zone: str :param availability_zone: EC2 availability zone of the cluster :type master_instance_type: str :param master_instance_type: EC2 instance type of the master :type slave_instance_type: str :param slave_instance_type: EC2 instance type of the slave nodes :type num_instances: int :param num_instances: Number of instances in the Hadoop cluster :type action_on_failure: str :param action_on_failure: Action to take if a step terminates :type keep_alive: bool :param keep_alive: Denotes whether the cluster should stay alive upon completion :type enable_debugging: bool :param enable_debugging: Denotes whether AWS console debugging should be enabled. :type hadoop_version: str :param hadoop_version: Version of Hadoop to use. This no longer defaults to '0.20' and now uses the AMI default. :type steps: list(boto.emr.Step) :param steps: List of steps to add with the job :type bootstrap_actions: list(boto.emr.BootstrapAction) :param bootstrap_actions: List of bootstrap actions that run before Hadoop starts. :type instance_groups: list(boto.emr.InstanceGroup) :param instance_groups: Optional list of instance groups to use when creating this job. NB: When provided, this argument supersedes num_instances and master/slave_instance_type. :type ami_version: str :param ami_version: Amazon Machine Image (AMI) version to use for instances. Values accepted by EMR are '1.0', '2.0', and 'latest'; EMR currently defaults to '1.0' if you don't set 'ami_version'. :type additional_info: JSON str :param additional_info: A JSON string for selecting additional features :type api_params: dict :param api_params: a dictionary of additional parameters to pass directly to the EMR API (so you don't have to upgrade boto to use new EMR features). You can also delete an API parameter by setting it to None. :type visible_to_all_users: bool :param visible_to_all_users: Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. If this value is set to ``True``, all IAM users of that AWS account can view and (if they have the proper policy permissions set) manage the job flow. If it is set to ``False``, only the IAM user that created the job flow can view and manage it. :type job_flow_role: str :param job_flow_role: An IAM role for the job flow. The EC2 instances of the job flow assume this role. The default role is ``EMRJobflowDefault``. In order to use the default role, you must have already created it using the CLI. :type service_role: str :param service_role: The IAM role that will be assumed by the Amazon EMR service to access AWS resources on your behalf. :rtype: str :return: The jobflow id """ params = {} if action_on_failure: params['ActionOnFailure'] = action_on_failure if log_uri: params['LogUri'] = log_uri params['Name'] = name # Common instance args common_params = self._build_instance_common_args(ec2_keyname, availability_zone, keep_alive, hadoop_version) params.update(common_params) # NB: according to the AWS API's error message, we must # "configure instances either using instance count, master and # slave instance type or instance groups but not both." # # Thus we switch here on the truthiness of instance_groups. if not instance_groups: # Instance args (the common case) instance_params = self._build_instance_count_and_type_args( master_instance_type, slave_instance_type, num_instances) params.update(instance_params) else: # Instance group args (for spot instances or a heterogenous cluster) list_args = self._build_instance_group_list_args(instance_groups) instance_params = dict( ('Instances.%s' % k, v) for k, v in six.iteritems(list_args) ) params.update(instance_params) # Debugging step from EMR API docs if enable_debugging: debugging_step = JarStep(name='Setup Hadoop Debugging', action_on_failure='TERMINATE_JOB_FLOW', main_class=None, jar=self.DebuggingJar, step_args=self.DebuggingArgs) steps.insert(0, debugging_step) # Step args if steps: step_args = [self._build_step_args(step) for step in steps] params.update(self._build_step_list(step_args)) if bootstrap_actions: bootstrap_action_args = [self._build_bootstrap_action_args(bootstrap_action) for bootstrap_action in bootstrap_actions] params.update(self._build_bootstrap_action_list(bootstrap_action_args)) if ami_version: params['AmiVersion'] = ami_version if additional_info is not None: params['AdditionalInfo'] = additional_info if api_params: for key, value in six.iteritems(api_params): if value is None: params.pop(key, None) else: params[key] = value if visible_to_all_users is not None: if visible_to_all_users: params['VisibleToAllUsers'] = 'true' else: params['VisibleToAllUsers'] = 'false' if job_flow_role is not None: params['JobFlowRole'] = job_flow_role if service_role is not None: params['ServiceRole'] = service_role response = self.get_object( 'RunJobFlow', params, RunJobFlowResponse, verb='POST') return response.jobflowid