def _getClient(self): """ Build a txAWS S3 client using our stored credentials. """ creds = AWSCredentials(access_key=self.accessKey.encode('utf-8'), secret_key=self.secretKey.encode('utf-8')) region = AWSServiceRegion(creds=creds) return region.get_s3_client()
def _getClient(self): """ Build a txAWS S3 client using our stored credentials. """ creds = AWSCredentials( access_key=self.accessKey.encode('utf-8'), secret_key=self.secretKey.encode('utf-8')) region = AWSServiceRegion(creds=creds) return region.get_s3_client()
def __init__(self, engine): self.engine = engine self.s3conn = boto.connect_s3( aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY ) if not settings.USE_BOTO: self.txs3conn = AWSServiceRegion( access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, s3_uri=S3_US[0]['endpoint'], # s3_uri='https://s3.amazonaws.com', ).get_s3_client() self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE)
def __init__(self, environment_name, config): super(MachineProvider, self).__init__(environment_name, config) if not config.get("ec2-uri"): ec2_uri = get_region_uri(config.get("region", "us-east-1")) else: ec2_uri = config.get("ec2-uri") self._service = AWSServiceRegion( access_key=config.get("access-key", ""), secret_key=config.get("secret-key", ""), ec2_uri=ec2_uri, s3_uri=config.get("s3-uri", "")) self.s3 = self._service.get_s3_client() self.ec2 = self._service.get_ec2_client()
def main(reactor): aws = AWSServiceRegion(AWSCredentials()) agent = Agent(reactor) route53 = get_route53_client(agent, aws) d = route53.list_resource_record_sets(zone_id="Z2T2TSJ409GHZ9") d.addCallback(pprint) return d
def __init__(self, environment_name, config): super(MachineProvider, self).__init__(environment_name, config) if not config.get("ec2-uri"): ec2_uri = get_region_uri(config.get("region", DEFAULT_REGION)) else: ec2_uri = config.get("ec2-uri") self._service = AWSServiceRegion( access_key=config.get("access-key", ""), secret_key=config.get("secret-key", ""), ec2_uri=ec2_uri, s3_uri=config.get("s3-uri", "")) ssl_verify = self.config.get("ssl-hostname-verification", False) if ssl and ssl_verify: self._service.ec2_endpoint.ssl_hostname_verification = True self._service.s3_endpoint.ssl_hostname_verification = True elif ssl: log.warn('ssl-hostname-verification is disabled for this environment') else: log.warn('txaws.client.ssl unavailable for SSL hostname verification') ssl_verify = False for endpoint, endpoint_type in [(self._service.ec2_endpoint,'EC2'), (self._service.s3_endpoint,'S3')]: if endpoint.scheme != 'https': log.warn('%s API calls not using secure transport' % endpoint_type) elif not ssl_verify: log.warn('%s API calls encrypted but not authenticated' % endpoint_type) if not ssl_verify: log.warn('Ubuntu Cloud Image lookups encrypted but not authenticated') self.s3 = self._service.get_s3_client() self.ec2 = self._service.get_ec2_client()
def test_error_changes(self): duplicate_resource = POSTableData( sample_create_resource_record_sets_error_result.xml, b"text/xml", BAD_REQUEST, ) zone_id = "1234ABCDEF" agent = RequestTraversalAgent( static_resource({ b"2013-04-01": { b"hostedzone": { zone_id.encode("ascii"): { b"rrset": duplicate_resource, }, }, }, })) aws = AWSServiceRegion(access_key="abc", secret_key="def") client = get_route53_client(agent, aws, uncooperator()) err = self.failureResultOf( client.change_resource_record_sets( zone_id=zone_id, changes=[ create_rrset( sample_create_resource_record_sets_error_result.rrset) ], ), Route53Error) expected = { 'Code': 'InvalidChangeBatch', 'Message': "[Tried to create resource record set [name='duplicate.example.invalid.', type='CNAME'] but it already exists]", 'Type': 'Sender', } self.assertEqual(err.value.errors, [expected])
def test_creation_with_keys_and_creds(self): """ creds take precedence over individual access key/secret key pairs. """ region = AWSServiceRegion(self.creds, access_key="baz", secret_key="quux") self.assertEquals(region.creds.access_key, "foo") self.assertEquals(region.creds.secret_key, "bar")
def __init__(self, engine): self.engine = engine self.s3conn = boto.connect_s3( aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY ) self.txs3conn = AWSServiceRegion( access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, s3_uri=S3_EU_WEST[0]['endpoint'], ).get_s3_client() self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE)
def __init__(self, engine): self.engine = engine self.s3conn = boto.connect_s3( aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY ) if not settings.USE_BOTO: self.txs3conn = AWSServiceRegion( access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, s3_uri=settings.AWS_S3_ENDPOINT, ).get_s3_client() self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE)
def __init__(self, reactor): from txaws.service import AWSServiceRegion gtk.StatusIcon.__init__(self) self.set_from_stock(gtk.STOCK_NETWORK) self.set_visible(True) self.reactor = reactor self.connect("activate", self.on_activate) self.probing = False # Nested import because otherwise we get "reactor already installed". self.password_dialog = None try: creds = AWSCredentials() except ValueError: creds = self.from_gnomekeyring() self.region = AWSServiceRegion(creds) self.create_client(creds) menu = """ <ui> <menubar name="Menubar"> <menu action="Menu"> <menuitem action="Stop instances"/> </menu> </menubar> </ui> """ actions = [ ("Menu", None, "Menu"), ("Stop instances", gtk.STOCK_STOP, "_Stop instances...", None, "Stop instances", self.on_stop_instances), ] ag = gtk.ActionGroup("Actions") ag.add_actions(actions) self.manager = gtk.UIManager() self.manager.insert_action_group(ag, 0) self.manager.add_ui_from_string(menu) self.menu = self.manager.get_widget( "/Menubar/Menu/Stop instances").props.parent self.connect("popup-menu", self.on_popup_menu)
def __init__(self, engine): self.engine = engine self.s3conn = boto3.Session( aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name=settings.AWS_REGION ) if not settings.USE_BOTO: self.txs3conn = AWSServiceRegion( access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, s3_uri=S3_EU_WEST[0]['endpoint'], ).get_s3_client() self.s3resource = self.s3conn.resource('s3')
def test_some_zones(self): agent = RequestTraversalAgent( static_resource({ b"2013-04-01": { b"hostedzone": Data( sample_list_hosted_zones_result.xml, b"text/xml", ), }, })) aws = AWSServiceRegion(access_key="abc", secret_key="def") client = get_route53_client(agent, aws, uncooperator()) zones = self.successResultOf(client.list_hosted_zones()) expected = [HostedZone(**sample_list_hosted_zones_result.details)] self.assertEquals(expected, zones)
def _client_for_rrsets(self, zone_id, rrsets_xml): agent = RequestTraversalAgent( static_resource({ b"2013-04-01": { b"hostedzone": { zone_id: { b"rrset": Data( rrsets_xml, b"text/xml", ) } } } })) aws = AWSServiceRegion(access_key="abc", secret_key="def") return get_route53_client(agent, aws, uncooperator())
def _finish_convergence_service(k8s_client, options, subscription_client): k8s = KubeClient(k8s=k8s_client) access_key_id = FilePath( options["aws-access-key-id-path"]).getContent().strip() secret_access_key = FilePath( options["aws-secret-access-key-path"]).getContent().strip() aws = AWSServiceRegion(creds=AWSCredentials( access_key=access_key_id, secret_key=secret_access_key, )) Message.log( event=u"convergence-service:key-notification", key_id=access_key_id.decode("ascii"), secret_key_hash=sha256(secret_access_key).hexdigest().decode("ascii"), ) # XXX I get to leave a ton of fields empty because I happen to know # they're not used in this codepath. :/ Maybe this suggests something has # gone wrong ... config = DeploymentConfiguration( domain=options["domain"].decode("ascii"), kubernetes_namespace=options["kubernetes-namespace"].decode("ascii"), subscription_manager_endpoint=URL.fromText( options["endpoint"].decode("ascii")), s3_access_key_id=access_key_id.decode("ascii"), s3_secret_key=secret_access_key.decode("ascii"), introducer_image=options["introducer-image"].decode("ascii"), storageserver_image=options["storageserver-image"].decode("ascii"), log_gatherer_furl=None, stats_gatherer_furl=None, ) return TimerService( options["interval"], divert_errors_to_log(converge, u"subscription_converger"), config, subscription_client, k8s, aws, )
def _finish_convergence_service( k8s_client, options, subscription_client, reactor, ): k8s = KubeClient(k8s=k8s_client) access_key_id = FilePath(options["aws-access-key-id-path"]).getContent().strip() secret_access_key = FilePath(options["aws-secret-access-key-path"]).getContent().strip() aws = AWSServiceRegion(creds=AWSCredentials( access_key=access_key_id, secret_key=secret_access_key, )) Message.log( event=u"convergence-service:key-notification", key_id=access_key_id.decode("ascii"), secret_key_hash=sha256(secret_access_key).hexdigest().decode("ascii"), ) config = DeploymentConfiguration( domain=options["domain"].decode("ascii"), kubernetes_namespace=options["kubernetes-namespace"].decode("ascii"), subscription_manager_endpoint=URL.fromText(options["endpoint"].decode("ascii")), s3_access_key_id=access_key_id.decode("ascii"), s3_secret_key=secret_access_key.decode("ascii"), introducer_image=options["introducer-image"].decode("ascii"), storageserver_image=options["storageserver-image"].decode("ascii"), log_gatherer_furl=options["log-gatherer-furl"], stats_gatherer_furl=options["stats-gatherer-furl"], ) return _convergence_service( reactor, options["interval"], config, subscription_client, k8s, aws, )
def run(self): """ Run the configured method and write the HTTP response status and text to the output stream. """ region = AWSServiceRegion(access_key=self.key, secret_key=self.secret, uri=self.endpoint) query = self.query_factory(action=self.action, creds=region.creds, endpoint=region.ec2_endpoint, other_params=self.parameters) def write_response(response): print >> self.output, "URL: %s" % query.client.url print >> self.output print >> self.output, "HTTP status code: %s" % query.client.status print >> self.output print >> self.output, response def write_error(failure): if failure.check(AWSError): message = failure.value.original else: message = failure.getErrorMessage() if message.startswith("Error Message: "): message = message[len("Error Message: "):] print >> self.output, "URL: %s" % query.client.url print >> self.output if getattr(query.client, "status", None) is not None: print >> self.output, "HTTP status code: %s" % ( query.client.status,) print >> self.output print >> self.output, message if getattr(failure.value, "response", None) is not None: print >> self.output print >> self.output, failure.value.response deferred = query.submit() deferred.addCallback(write_response) deferred.addErrback(write_error) return deferred
def test_some_changes(self): change_resource = POSTableData( sample_change_resource_record_sets_result.xml, b"text/xml", ) zone_id = u"ABCDEF1234" agent = RequestTraversalAgent( static_resource({ b"2013-04-01": { b"hostedzone": { zone_id.encode("ascii"): { b"rrset": change_resource, } }, }, })) aws = AWSServiceRegion(access_key="abc", secret_key="def") client = get_route53_client(agent, aws, uncooperator()) self.successResultOf( client.change_resource_record_sets( zone_id=zone_id, changes=[ create_rrset( sample_change_resource_record_sets_result.rrset), delete_rrset( sample_change_resource_record_sets_result.rrset), upsert_rrset( sample_change_resource_record_sets_result.rrset), ], )) # Ack, what a pathetic assertion. change_template = u"<Change><Action>{action}</Action><ResourceRecordSet><Name>example.invalid.</Name><Type>NS</Type><TTL>86400</TTL><ResourceRecords><ResourceRecord><Value>ns1.example.invalid.</Value></ResourceRecord><ResourceRecord><Value>ns2.example.invalid.</Value></ResourceRecord></ResourceRecords></ResourceRecordSet></Change>" changes = [ change_template.format(action=u"CREATE"), change_template.format(action=u"DELETE"), change_template.format(action=u"UPSERT"), ] expected = u"""\ <?xml version="1.0" encoding="UTF-8"?> <ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ChangeBatch><Changes>{changes}</Changes></ChangeBatch></ChangeResourceRecordSetsRequest>""".format( changes=u"".join(changes)).encode("utf-8") self.assertEqual((expected, ), change_resource.posted)
class AWSStatusIcon(gtk.StatusIcon): """A status icon shown when instances are running.""" def __init__(self, reactor): from txaws.service import AWSServiceRegion gtk.StatusIcon.__init__(self) self.set_from_stock(gtk.STOCK_NETWORK) self.set_visible(True) self.reactor = reactor self.connect("activate", self.on_activate) self.probing = False # Nested import because otherwise we get "reactor already installed". self.password_dialog = None try: creds = AWSCredentials() except ValueError: creds = self.from_gnomekeyring() self.region = AWSServiceRegion(creds) self.create_client(creds) menu = """ <ui> <menubar name="Menubar"> <menu action="Menu"> <menuitem action="Stop instances"/> </menu> </menubar> </ui> """ actions = [ ("Menu", None, "Menu"), ("Stop instances", gtk.STOCK_STOP, "_Stop instances...", None, "Stop instances", self.on_stop_instances), ] ag = gtk.ActionGroup("Actions") ag.add_actions(actions) self.manager = gtk.UIManager() self.manager.insert_action_group(ag, 0) self.manager.add_ui_from_string(menu) self.menu = self.manager.get_widget( "/Menubar/Menu/Stop instances").props.parent self.connect("popup-menu", self.on_popup_menu) def create_client(self, creds): if creds is not None: self.client = self.region.get_ec2_client() self.on_activate(None) else: # waiting on user entered credentials. self.client = None def from_gnomekeyring(self): # Try for gtk gui specific credentials. try: items = gnomekeyring.find_items_sync( gnomekeyring.ITEM_GENERIC_SECRET, { "aws-host": "aws.amazon.com", }) except (gnomekeyring.NoMatchError, gnomekeyring.DeniedError): self.show_a_password_dialog() return None else: key_id, secret_key = items[0].secret.split(":") return AWSCredentials(access_key=key_id, secret_key=secret_key) def show_a_password_dialog(self): self.password_dialog = gtk.Dialog( "Enter your AWS credentals", None, gtk.DIALOG_MODAL, (gtk.STOCK_OK, gtk.RESPONSE_ACCEPT, gtk.STOCK_CANCEL, gtk.RESPONSE_REJECT)) content = self.password_dialog.get_content_area() def add_entry(name): box = gtk.HBox() box.show() content.add(box) label = gtk.Label(name) label.show() box.add(label) entry = gtk.Entry() entry.show() box.add(entry) label.set_use_underline(True) label.set_mnemonic_widget(entry) add_entry("AWS _Access Key ID") add_entry("AWS _Secret Key") self.password_dialog.show() self.password_dialog.connect("response", self.save_key) self.password_dialog.run() def on_activate(self, data): if self.probing or not self.client: # don't ask multiple times, and don't ask until we have # credentials. return self.probing = True deferred = self.client.describe_instances() deferred.addCallbacks(self.showhide, self.describe_error) def on_popup_menu(self, status, button, time): self.menu.popup(None, None, None, button, time) def on_stop_instances(self, data): # It would be nice to popup a window to select instances.. TODO. deferred = self.client.describe_instances() deferred.addCallbacks(self.shutdown_instances, self.show_error) def save_key(self, response_id, data): try: if data != gtk.RESPONSE_ACCEPT: # User cancelled. They can ask for the password again somehow. return content = self.password_dialog.get_content_area() key_id = content.get_children()[0].get_children()[1].get_text() secret_key = content.get_children()[1].get_children()[1].get_text() creds = AWSCredentials(access_key=key_id, secret_key=secret_key) self.create_client(creds) gnomekeyring.item_create_sync( None, gnomekeyring.ITEM_GENERIC_SECRET, "AWS access credentials", {"aws-host": "aws.amazon.com"}, "%s:%s" % (key_id, secret_key), True) finally: self.password_dialog.hide() # XXX? Does this leak? self.password_dialog = None def showhide(self, reservation): active = 0 for instance in reservation: if instance.instance_state == "running": active += 1 self.set_tooltip("AWS Status - %d instances" % active) self.set_visible(active != 0) self.queue_check() def shutdown_instances(self, reservation): d = self.client.terminate_instances( *[instance.instance_id for instance in reservation]) d.addCallbacks(self.on_activate, self.show_error) def queue_check(self): self.probing = False self.reactor.callLater(60, self.on_activate, None) def show_error(self, error): # debugging output for now. print error.value try: print error.value.response except: pass def describe_error(self, error): from twisted.internet.defer import TimeoutError if isinstance(error.value, TimeoutError): # timeout errors can be ignored - transient network issue or some # such. pass else: # debugging output for now. self.show_error(error) self.queue_check()
def setUp(self): region = AWSServiceRegion() self.ec2 = region.get_ec2_client() self.s3 = region.get_s3_client()
def setUp(self): self.creds = AWSCredentials("foo", "bar") self.region = AWSServiceRegion(creds=self.creds)
class MachineProvider(MachineProviderBase): """MachineProvider for use in an EC2/S3 environment""" def __init__(self, environment_name, config): super(MachineProvider, self).__init__(environment_name, config) if not config.get("ec2-uri"): ec2_uri = get_region_uri(config.get("region", DEFAULT_REGION)) else: ec2_uri = config.get("ec2-uri") self._service = AWSServiceRegion( access_key=config.get("access-key", ""), secret_key=config.get("secret-key", ""), ec2_uri=ec2_uri, s3_uri=config.get("s3-uri", "")) ssl_verify = self.config.get("ssl-hostname-verification", False) if ssl and ssl_verify: self._service.ec2_endpoint.ssl_hostname_verification = True self._service.s3_endpoint.ssl_hostname_verification = True elif ssl: log.warn('ssl-hostname-verification is disabled for this environment') else: log.warn('txaws.client.ssl unavailable for SSL hostname verification') ssl_verify = False for endpoint, endpoint_type in [(self._service.ec2_endpoint,'EC2'), (self._service.s3_endpoint,'S3')]: if endpoint.scheme != 'https': log.warn('%s API calls not using secure transport' % endpoint_type) elif not ssl_verify: log.warn('%s API calls encrypted but not authenticated' % endpoint_type) if not ssl_verify: log.warn('Ubuntu Cloud Image lookups encrypted but not authenticated') self.s3 = self._service.get_s3_client() self.ec2 = self._service.get_ec2_client() @property def provider_type(self): return "ec2" @property def using_amazon(self): return "ec2-uri" not in self.config @inlineCallbacks def get_constraint_set(self): """Return the set of constraints that are valid for this provider.""" cs = yield super(MachineProvider, self).get_constraint_set() if 1: # These keys still need to be valid (instance-type and ec2-zone) #if self.using_amazon: # Expose EC2 instance types/zones on AWS itelf, not private clouds. cs.register_generics(INSTANCE_TYPES.keys()) cs.register("ec2-zone", converter=convert_zone) returnValue(cs) def get_legacy_config_keys(self): """Return any deprecated config keys that are set""" legacy = super(MachineProvider, self).get_legacy_config_keys() if self.using_amazon: # In the absence of a generic instance-type/image-id mechanism, # these keys remain valid on private clouds. amazon_legacy = set(("default-image-id", "default-instance-type")) legacy.update(amazon_legacy.intersection(self.config)) return legacy def get_serialization_data(self): """Get provider configuration suitable for serialization. Also extracts credential information from the environment. """ data = super(MachineProvider, self).get_serialization_data() data.setdefault("access-key", os.environ.get("AWS_ACCESS_KEY_ID")) data.setdefault("secret-key", os.environ.get("AWS_SECRET_ACCESS_KEY")) return data def get_file_storage(self): """Retrieve an S3-backed :class:`FileStorage`.""" return FileStorage(self.s3, self.config["control-bucket"]) def start_machine(self, machine_data, master=False): """Start an EC2 machine. :param dict machine_data: desired characteristics of the new machine; it must include a "machine-id" key, and may include a "constraints" key to specify the underlying OS and hardware. :param bool master: if True, machine will initialize the juju admin and run a provisioning agent, in addition to running a machine agent. """ return EC2LaunchMachine.launch(self, machine_data, master) @inlineCallbacks def get_machines(self, instance_ids=()): """List machines running in the provider. :param list instance_ids: ids of instances you want to get. Leave empty to list every :class:`juju.providers.ec2.machine.EC2ProviderMachine` owned by this provider. :return: a list of :class:`juju.providers.ec2.machine.EC2ProviderMachine` instances :rtype: :class:`twisted.internet.defer.Deferred` :raises: :exc:`juju.errors.MachinesNotFound` """ group_name = "juju-%s" % self.environment_name try: instances = yield self.ec2.describe_instances(*instance_ids) except EC2Error as error: code = error.get_error_codes() message = error.get_error_messages() if code == "InvalidInstanceID.NotFound": message = error.get_error_messages() raise MachinesNotFound( re.findall(r"\bi-[0-9a-f]{3,15}\b", message)) raise ProviderInteractionError( "Unexpected EC2Error getting machines %s: %s" % (", ".join(instance_ids), message)) machines = [] for instance in instances: if instance.instance_state not in ("running", "pending"): continue if group_name not in instance.reservation.groups: continue machines.append(machine_from_instance(instance)) if instance_ids: # We were asked for a specific list of machines, and if we can't # completely fulfil that request we should blow up. found_instance_ids = set(m.instance_id for m in machines) missing = set(instance_ids) - found_instance_ids if missing: raise MachinesNotFound(missing) returnValue(machines) @inlineCallbacks def destroy_environment(self): """Terminate all associated machines and security groups. The super defintion of this method terminates each machine in the environment; this needs to be augmented here by also removing the security group for the environment. :rtype: :class:`twisted.internet.defer.Deferred` """ try: killed_machines = yield super(MachineProvider, self).\ destroy_environment() returnValue(killed_machines) finally: yield destroy_environment_security_group(self) @inlineCallbacks def shutdown_machines(self, machines): """Terminate machines associated with this provider. :param machines: machines to shut down :type machines: list of :class:`juju.providers.ec2.machine.EC2ProviderMachine` :return: list of terminated :class:`juju.providers.ec2.machine.EC2ProviderMachine` instances :rtype: :class:`twisted.internet.defer.Deferred` """ if not machines: returnValue([]) for machine in machines: if not isinstance(machine, EC2ProviderMachine): raise ProviderError("Can only shut down EC2ProviderMachines; " "got a %r" % type(machine)) ids = [m.instance_id for m in machines] killable_machines = yield self.get_machines(ids) if not killable_machines: returnValue([]) # Nothing to do killable_ids = [m.instance_id for m in killable_machines] terminated = yield self.ec2.terminate_instances(*killable_ids) # Pass on what was actually terminated, in the case the # machine has somehow disappeared since get_machines # above. This is to avoid getting EC2Error: Error Message: # Invalid id when running ec2.describe_instances in # remove_security_groups terminated_ids = [info[0] for info in terminated] yield remove_security_groups(self, terminated_ids) returnValue(killable_machines) def open_port(self, machine, machine_id, port, protocol="tcp"): """Authorizes `port` using `protocol` on EC2 for `machine`.""" return open_provider_port(self, machine, machine_id, port, protocol) def close_port(self, machine, machine_id, port, protocol="tcp"): """Revokes `port` using `protocol` on EC2 for `machine`.""" return close_provider_port(self, machine, machine_id, port, protocol) def get_opened_ports(self, machine, machine_id): """Returns a set of open (port, proto) pairs for `machine`.""" return get_provider_opened_ports(self, machine, machine_id)
class S3Downloader(object): """ Pipeline process which downloads a media file from S3 """ def __init__(self, engine): self.engine = engine self.s3conn = boto.connect_s3( aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY ) if not settings.USE_BOTO: self.txs3conn = AWSServiceRegion( access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, s3_uri=S3_US[0]['endpoint'], # s3_uri='https://s3.amazonaws.com', ).get_s3_client() self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE) @defer.inlineCallbacks def _get_data_from_s3_tx(self, path): """ txAWS GET from S3 """ image = yield self.txs3conn.get_object( settings.IMAGES_STORE, str(path), ) defer.returnValue(image) def _get_data_from_s3(self, path): """ boto GET from S3 """ key = self.botobucket.get_key(path) data = key.get_contents_as_string() return data @time_on_statsd(statsd_name(), 's3_downloader') def process_image(self, payload, **kwargs): """ Gets image data from S3. This attempts to download from s3 settings.ATTEMPTS times and timeouts after settings.S3_TIMEOUT """ def _create_deferred(timeout=0): """ Creates a deferred which will run after a given delay """ if settings.USE_BOTO: dfd = task.deferLater( reactor, timeout, threads.deferToThread, self._get_data_from_s3, payload['image_path'] ) return dfd else: dfd = task.deferLater( reactor, timeout, self._get_data_from_s3_tx, payload['image_path'] ) return dfd def _s3callback(deferred_list_result): """ When one of the requests has completed, cancel the rest """ [dfd.cancel() for dfd in dfds_list if not dfd.called] if not deferred_list_result[0]: raise NoDataInS3Error() payload['original_image'] = deferred_list_result[0] return payload def _timeout_and_fail(dfds_list): """ If none of the defers has finished by (attempts+1)*timeout then cancel and return an error """ [dfd.cancel() for dfd in dfds_list if not dfd.called] def _surpress_cancel_error(result): if isinstance(result, defer.CancelledError): pass # Skip if already exists from cache if 'original_image' in payload.keys(): return payload if settings.DEBUG: log.msg( "[%s] Starting S3 Download" % datetime.now().isoformat(), logLevel=logging.DEBUG ) # Make a deferred list of download attempts that have their predefined starting # times baked into the deferred. Return when any deferred has a successful result # Keep a list of the original deferred as we cannot access them once in DeferredList dfds_list = [] for attempt in range(0, settings.S3_ATTEMPTS): dfds_list.append(_create_deferred(timeout=attempt*settings.S3_TIMEOUT)) dfds_list[-1].addErrback(_surpress_cancel_error) dfds = defer.DeferredList(dfds_list, fireOnOneCallback=True) dfds.addCallback(_s3callback) # Auto cancel requests which don't fire after their max timeout reactor.callLater( settings.S3_ATTEMPTS*settings.S3_TIMEOUT, _timeout_and_fail, dfds_list ) return dfds
class MachineProvider(MachineProviderBase): """MachineProvider for use in an EC2/S3 environment""" def __init__(self, environment_name, config): super(MachineProvider, self).__init__(environment_name, config) if not config.get("ec2-uri"): ec2_uri = get_region_uri(config.get("region", "us-east-1")) else: ec2_uri = config.get("ec2-uri") self._service = AWSServiceRegion( access_key=config.get("access-key", ""), secret_key=config.get("secret-key", ""), ec2_uri=ec2_uri, s3_uri=config.get("s3-uri", "")) self.s3 = self._service.get_s3_client() self.ec2 = self._service.get_ec2_client() @property def provider_type(self): return "ec2" def get_serialization_data(self): """Get provider configuration suitable for serialization. Also extracts credential information from the environment. """ data = super(MachineProvider, self).get_serialization_data() data.setdefault("access-key", os.environ.get("AWS_ACCESS_KEY_ID")) data.setdefault("secret-key", os.environ.get("AWS_SECRET_ACCESS_KEY")) return data def get_file_storage(self): """Retrieve an S3-backed :class:`FileStorage`.""" return FileStorage(self.s3, self.config["control-bucket"]) def start_machine(self, machine_data, master=False): """Start an EC2 machine. :param dict machine_data: desired characteristics of the new machine; it must include a "machine-id" key, and may include a "constraints" key to specify the underlying OS and hardware. :param bool master: if True, machine will initialize the juju admin and run a provisioning agent, in addition to running a machine agent. """ if "machine-id" not in machine_data: return fail(ProviderError( "Cannot launch a machine without specifying a machine-id")) machine_id = machine_data["machine-id"] constraints = machine_data.get("constraints", {}) return EC2LaunchMachine(self, master, constraints).run(machine_id) @inlineCallbacks def get_machines(self, instance_ids=()): """List machines running in the provider. :param list instance_ids: ids of instances you want to get. Leave empty to list every :class:`juju.providers.ec2.machine.EC2ProviderMachine` owned by this provider. :return: a list of :class:`juju.providers.ec2.machine.EC2ProviderMachine` instances :rtype: :class:`twisted.internet.defer.Deferred` :raises: :exc:`juju.errors.MachinesNotFound` """ group_name = "juju-%s" % self.environment_name try: instances = yield self.ec2.describe_instances(*instance_ids) except EC2Error as error: code = error.get_error_codes() message = error.get_error_messages() if code == "InvalidInstanceID.NotFound": message = error.get_error_messages() raise MachinesNotFound( re.findall(r"\bi-[0-9a-f]{3,15}\b", message)) raise ProviderInteractionError( "Unexpected EC2Error getting machines %s: %s" % (", ".join(instance_ids), message)) machines = [] for instance in instances: if instance.instance_state not in ("running", "pending"): continue if group_name not in instance.reservation.groups: continue machines.append(machine_from_instance(instance)) if instance_ids: # We were asked for a specific list of machines, and if we can't # completely fulfil that request we should blow up. found_instance_ids = set(m.instance_id for m in machines) missing = set(instance_ids) - found_instance_ids if missing: raise MachinesNotFound(missing) returnValue(machines) @inlineCallbacks def destroy_environment(self): """Terminate all associated machines and security groups. The super defintion of this method terminates each machine in the environment; this needs to be augmented here by also removing the security group for the environment. :rtype: :class:`twisted.internet.defer.Deferred` """ try: killed_machines = yield super(MachineProvider, self).\ destroy_environment() returnValue(killed_machines) finally: yield destroy_environment_security_group(self) @inlineCallbacks def shutdown_machines(self, machines): """Terminate machines associated with this provider. :param machines: machines to shut down :type machines: list of :class:`juju.providers.ec2.machine.EC2ProviderMachine` :return: list of terminated :class:`juju.providers.ec2.machine.EC2ProviderMachine` instances :rtype: :class:`twisted.internet.defer.Deferred` """ if not machines: returnValue([]) for machine in machines: if not isinstance(machine, EC2ProviderMachine): raise ProviderError("Can only shut down EC2ProviderMachines; " "got a %r" % type(machine)) ids = [m.instance_id for m in machines] killable_machines = yield self.get_machines(ids) if not killable_machines: returnValue([]) # Nothing to do killable_ids = [m.instance_id for m in killable_machines] terminated = yield self.ec2.terminate_instances(*killable_ids) # Pass on what was actually terminated, in the case the # machine has somehow disappeared since get_machines # above. This is to avoid getting EC2Error: Error Message: # Invalid id when running ec2.describe_instances in # remove_security_groups terminated_ids = [info[0] for info in terminated] yield remove_security_groups(self, terminated_ids) returnValue(killable_machines) def open_port(self, machine, machine_id, port, protocol="tcp"): """Authorizes `port` using `protocol` on EC2 for `machine`.""" return open_provider_port(self, machine, machine_id, port, protocol) def close_port(self, machine, machine_id, port, protocol="tcp"): """Revokes `port` using `protocol` on EC2 for `machine`.""" return close_provider_port(self, machine, machine_id, port, protocol) def get_opened_ports(self, machine, machine_id): """Returns a set of open (port, proto) pairs for `machine`.""" return get_provider_opened_ports(self, machine, machine_id)
class AWSServiceRegionTestCase(TestCase): def setUp(self): self.creds = AWSCredentials("foo", "bar") self.region = AWSServiceRegion(creds=self.creds) def test_simple_creation(self): self.assertEquals(self.creds, self.region.creds) self.assertEquals(self.region._clients, {}) self.assertEquals(self.region.ec2_endpoint.get_uri(), EC2_ENDPOINT_US) def test_creation_with_keys(self): region = AWSServiceRegion(access_key="baz", secret_key="quux") self.assertEquals(region.creds.access_key, "baz") self.assertEquals(region.creds.secret_key, "quux") def test_creation_with_keys_and_creds(self): """ creds take precedence over individual access key/secret key pairs. """ region = AWSServiceRegion(self.creds, access_key="baz", secret_key="quux") self.assertEquals(region.creds.access_key, "foo") self.assertEquals(region.creds.secret_key, "bar") def test_creation_with_uri(self): region = AWSServiceRegion( creds=self.creds, ec2_uri="http://foo/bar") self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar") def test_creation_with_uri_backwards_compatible(self): region = AWSServiceRegion( creds=self.creds, uri="http://foo/bar") self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar") def test_creation_with_uri_and_region(self): region = AWSServiceRegion( creds=self.creds, region=REGION_EU, ec2_uri="http://foo/bar") self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar") def test_creation_with_region_override(self): region = AWSServiceRegion(creds=self.creds, region=REGION_EU) self.assertEquals(region.ec2_endpoint.get_uri(), EC2_ENDPOINT_EU) def test_get_ec2_client_with_empty_cache(self): key = str(EC2Client) + str(self.creds) + str(self.region.ec2_endpoint) original_client = self.region._clients.get(key) new_client = self.region.get_client( EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint) self.assertEquals(original_client, None) self.assertTrue(isinstance(new_client, EC2Client)) self.assertNotEquals(original_client, new_client) def test_get_ec2_client_from_cache_default(self): client1 = self.region.get_ec2_client() client2 = self.region.get_ec2_client() self.assertTrue(isinstance(client1, EC2Client)) self.assertTrue(isinstance(client2, EC2Client)) self.assertEquals(client1, client2) def test_get_ec2_client_from_cache(self): client1 = self.region.get_client( EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint) client2 = self.region.get_client( EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint) self.assertTrue(isinstance(client1, EC2Client)) self.assertTrue(isinstance(client2, EC2Client)) self.assertEquals(client1, client2) def test_get_ec2_client_from_cache_with_purge(self): client1 = self.region.get_client( EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint, purge_cache=True) client2 = self.region.get_client( EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint, purge_cache=True) self.assertTrue(isinstance(client1, EC2Client)) self.assertTrue(isinstance(client2, EC2Client)) self.assertNotEquals(client1, client2) def test_get_s3_client_with_empty_cache(self): key = str(S3Client) + str(self.creds) + str(self.region.s3_endpoint) original_client = self.region._clients.get(key) new_client = self.region.get_client( S3Client, creds=self.creds, endpoint=self.region.s3_endpoint) self.assertEquals(original_client, None) self.assertTrue(isinstance(new_client, S3Client)) self.assertNotEquals(original_client, new_client) test_get_s3_client_with_empty_cache.skip = s3clientSkip
def test_creation_with_uri_and_region(self): region = AWSServiceRegion( creds=self.creds, region=REGION_EU, ec2_uri="http://foo/bar") self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_keys(self): region = AWSServiceRegion(access_key="baz", secret_key="quux") self.assertEquals(region.creds.access_key, "baz") self.assertEquals(region.creds.secret_key, "quux")
def set_region(self, creds): from txaws.service import AWSServiceRegion self.region = AWSServiceRegion(creds)
def test_creation_with_uri_backwards_compatible(self): region = AWSServiceRegion( creds=self.creds, uri="http://foo/bar") self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
# # - in case they leak out of the test suite somehow # # - in case the implementation is broken and does something destructive # # - in case malicious code is inserted somehow (eg, you run # tests on code submitted by another developer) # # As far as I can tell there's no way to isolate an API user # from _some_ of the parent account's S3 buckets. Therefore, # isolation probably involves registering a new top-level AWS # account and dedicating it to testing purposes. try: access_key = environ["TXAWS_INTEGRATION_AWS_ACCESS_KEY_ID"] secret_key = environ["TXAWS_INTEGRATION_AWS_SECRET_ACCESS_KEY"] except KeyError as e: case.skipTest("Missing {} environment variable.".format(e)) else: credentials = AWSCredentials( access_key=access_key, secret_key=secret_key, ) return AWSServiceRegion(credentials) def get_memory_service(case): return FakeAWSServiceRegion( access_key="fake access key", secret_key="fake secret key", )
def test_creation_with_region_override(self): region = AWSServiceRegion(creds=self.creds, region=REGION_EU) self.assertEquals(region.ec2_endpoint.get_uri(), EC2_ENDPOINT_EU)