class AWSServiceRegionTestCase(TestCase):
def setUp(self):
self.creds = AWSCredentials("foo", "bar")
self.region = AWSServiceRegion(creds=self.creds)
def test_simple_creation(self):
self.assertEquals(self.creds, self.region.creds)
self.assertEquals(self.region._clients, {})
self.assertEquals(self.region.ec2_endpoint.get_uri(), EC2_ENDPOINT_US)
def test_creation_with_keys(self):
region = AWSServiceRegion(access_key="baz", secret_key="quux")
self.assertEquals(region.creds.access_key, "baz")
self.assertEquals(region.creds.secret_key, "quux")
def test_creation_with_keys_and_creds(self):
"""
creds take precedence over individual access key/secret key pairs.
"""
region = AWSServiceRegion(self.creds, access_key="baz",
secret_key="quux")
self.assertEquals(region.creds.access_key, "foo")
self.assertEquals(region.creds.secret_key, "bar")
def test_creation_with_uri(self):
region = AWSServiceRegion(
creds=self.creds, ec2_uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_uri_backwards_compatible(self):
region = AWSServiceRegion(
creds=self.creds, uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_uri_and_region(self):
region = AWSServiceRegion(
creds=self.creds, region=REGION_EU, ec2_uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_region_override(self):
region = AWSServiceRegion(creds=self.creds, region=REGION_EU)
self.assertEquals(region.ec2_endpoint.get_uri(), EC2_ENDPOINT_EU)
def test_get_ec2_client_with_empty_cache(self):
key = str(EC2Client) + str(self.creds) + str(self.region.ec2_endpoint)
original_client = self.region._clients.get(key)
new_client = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint)
self.assertEquals(original_client, None)
self.assertTrue(isinstance(new_client, EC2Client))
self.assertNotEquals(original_client, new_client)
def test_get_ec2_client_from_cache_default(self):
client1 = self.region.get_ec2_client()
client2 = self.region.get_ec2_client()
self.assertTrue(isinstance(client1, EC2Client))
self.assertTrue(isinstance(client2, EC2Client))
self.assertEquals(client1, client2)
def test_get_ec2_client_from_cache(self):
client1 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint)
client2 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint)
self.assertTrue(isinstance(client1, EC2Client))
self.assertTrue(isinstance(client2, EC2Client))
self.assertEquals(client1, client2)
def test_get_ec2_client_from_cache_with_purge(self):
client1 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint,
purge_cache=True)
client2 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint,
purge_cache=True)
self.assertTrue(isinstance(client1, EC2Client))
self.assertTrue(isinstance(client2, EC2Client))
self.assertNotEquals(client1, client2)
def test_get_s3_client_with_empty_cache(self):
key = str(S3Client) + str(self.creds) + str(self.region.s3_endpoint)
original_client = self.region._clients.get(key)
new_client = self.region.get_client(
S3Client, creds=self.creds, endpoint=self.region.s3_endpoint)
self.assertEquals(original_client, None)
self.assertTrue(isinstance(new_client, S3Client))
self.assertNotEquals(original_client, new_client)
test_get_s3_client_with_empty_cache.skip = s3clientSkip
class AWSStatusIcon(gtk.StatusIcon):
"""A status icon shown when instances are running."""
def __init__(self, reactor):
from txaws.service import AWSServiceRegion
gtk.StatusIcon.__init__(self)
self.set_from_stock(gtk.STOCK_NETWORK)
self.set_visible(True)
self.reactor = reactor
self.connect("activate", self.on_activate)
self.probing = False
# Nested import because otherwise we get "reactor already installed".
self.password_dialog = None
try:
creds = AWSCredentials()
except ValueError:
creds = self.from_gnomekeyring()
self.region = AWSServiceRegion(creds)
self.create_client(creds)
menu = """
<ui>
<menubar name="Menubar">
<menu action="Menu">
<menuitem action="Stop instances"/>
</menu>
</menubar>
</ui>
"""
actions = [
("Menu", None, "Menu"),
("Stop instances", gtk.STOCK_STOP, "_Stop instances...", None,
"Stop instances", self.on_stop_instances),
]
ag = gtk.ActionGroup("Actions")
ag.add_actions(actions)
self.manager = gtk.UIManager()
self.manager.insert_action_group(ag, 0)
self.manager.add_ui_from_string(menu)
self.menu = self.manager.get_widget(
"/Menubar/Menu/Stop instances").props.parent
self.connect("popup-menu", self.on_popup_menu)
def create_client(self, creds):
if creds is not None:
self.client = self.region.get_ec2_client()
self.on_activate(None)
else:
# waiting on user entered credentials.
self.client = None
def from_gnomekeyring(self):
# Try for gtk gui specific credentials.
try:
items = gnomekeyring.find_items_sync(
gnomekeyring.ITEM_GENERIC_SECRET,
{
"aws-host": "aws.amazon.com",
})
except (gnomekeyring.NoMatchError,
gnomekeyring.DeniedError):
self.show_a_password_dialog()
return None
else:
key_id, secret_key = items[0].secret.split(":")
return AWSCredentials(access_key=key_id, secret_key=secret_key)
def show_a_password_dialog(self):
self.password_dialog = gtk.Dialog(
"Enter your AWS credentals", None, gtk.DIALOG_MODAL,
(gtk.STOCK_OK, gtk.RESPONSE_ACCEPT,
gtk.STOCK_CANCEL,
gtk.RESPONSE_REJECT))
content = self.password_dialog.get_content_area()
def add_entry(name):
box = gtk.HBox()
box.show()
content.add(box)
label = gtk.Label(name)
label.show()
box.add(label)
entry = gtk.Entry()
entry.show()
box.add(entry)
label.set_use_underline(True)
label.set_mnemonic_widget(entry)
add_entry("AWS _Access Key ID")
add_entry("AWS _Secret Key")
self.password_dialog.show()
self.password_dialog.connect("response", self.save_key)
self.password_dialog.run()
def on_activate(self, data):
if self.probing or not self.client:
# don't ask multiple times, and don't ask until we have
# credentials.
return
self.probing = True
deferred = self.client.describe_instances()
deferred.addCallbacks(self.showhide, self.describe_error)
def on_popup_menu(self, status, button, time):
self.menu.popup(None, None, None, button, time)
def on_stop_instances(self, data):
# It would be nice to popup a window to select instances.. TODO.
deferred = self.client.describe_instances()
deferred.addCallbacks(self.shutdown_instances, self.show_error)
def save_key(self, response_id, data):
try:
if data != gtk.RESPONSE_ACCEPT:
# User cancelled. They can ask for the password again somehow.
return
content = self.password_dialog.get_content_area()
key_id = content.get_children()[0].get_children()[1].get_text()
secret_key = content.get_children()[1].get_children()[1].get_text()
creds = AWSCredentials(access_key=key_id, secret_key=secret_key)
self.create_client(creds)
gnomekeyring.item_create_sync(
None,
gnomekeyring.ITEM_GENERIC_SECRET,
"AWS access credentials",
{"aws-host": "aws.amazon.com"},
"%s:%s" % (key_id, secret_key), True)
finally:
self.password_dialog.hide()
# XXX? Does this leak?
self.password_dialog = None
def showhide(self, reservation):
active = 0
for instance in reservation:
if instance.instance_state == "running":
active += 1
self.set_tooltip("AWS Status - %d instances" % active)
self.set_visible(active != 0)
self.queue_check()
def shutdown_instances(self, reservation):
d = self.client.terminate_instances(
*[instance.instance_id for instance in reservation])
d.addCallbacks(self.on_activate, self.show_error)
def queue_check(self):
self.probing = False
self.reactor.callLater(60, self.on_activate, None)
def show_error(self, error):
# debugging output for now.
print error.value
try:
print error.value.response
except:
pass
def describe_error(self, error):
from twisted.internet.defer import TimeoutError
if isinstance(error.value, TimeoutError):
# timeout errors can be ignored - transient network issue or some
# such.
pass
else:
# debugging output for now.
self.show_error(error)
self.queue_check()
class MachineProvider(MachineProviderBase):
"""MachineProvider for use in an EC2/S3 environment"""
def __init__(self, environment_name, config):
super(MachineProvider, self).__init__(environment_name, config)
if not config.get("ec2-uri"):
ec2_uri = get_region_uri(config.get("region", "us-east-1"))
else:
ec2_uri = config.get("ec2-uri")
self._service = AWSServiceRegion(
access_key=config.get("access-key", ""),
secret_key=config.get("secret-key", ""),
ec2_uri=ec2_uri,
s3_uri=config.get("s3-uri", ""))
self.s3 = self._service.get_s3_client()
self.ec2 = self._service.get_ec2_client()
@property
def provider_type(self):
return "ec2"
def get_serialization_data(self):
"""Get provider configuration suitable for serialization.
Also extracts credential information from the environment.
"""
data = super(MachineProvider, self).get_serialization_data()
data.setdefault("access-key", os.environ.get("AWS_ACCESS_KEY_ID"))
data.setdefault("secret-key", os.environ.get("AWS_SECRET_ACCESS_KEY"))
return data
def get_file_storage(self):
"""Retrieve an S3-backed :class:`FileStorage`."""
return FileStorage(self.s3, self.config["control-bucket"])
def start_machine(self, machine_data, master=False):
"""Start an EC2 machine.
:param dict machine_data: desired characteristics of the new machine;
it must include a "machine-id" key, and may include a "constraints"
key to specify the underlying OS and hardware.
:param bool master: if True, machine will initialize the juju admin
and run a provisioning agent, in addition to running a machine
agent.
"""
if "machine-id" not in machine_data:
return fail(ProviderError(
"Cannot launch a machine without specifying a machine-id"))
machine_id = machine_data["machine-id"]
constraints = machine_data.get("constraints", {})
return EC2LaunchMachine(self, master, constraints).run(machine_id)
@inlineCallbacks
def get_machines(self, instance_ids=()):
"""List machines running in the provider.
:param list instance_ids: ids of instances you want to get. Leave empty
to list every
:class:`juju.providers.ec2.machine.EC2ProviderMachine` owned by
this provider.
:return: a list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
:raises: :exc:`juju.errors.MachinesNotFound`
"""
group_name = "juju-%s" % self.environment_name
try:
instances = yield self.ec2.describe_instances(*instance_ids)
except EC2Error as error:
code = error.get_error_codes()
message = error.get_error_messages()
if code == "InvalidInstanceID.NotFound":
message = error.get_error_messages()
raise MachinesNotFound(
re.findall(r"\bi-[0-9a-f]{3,15}\b", message))
raise ProviderInteractionError(
"Unexpected EC2Error getting machines %s: %s"
% (", ".join(instance_ids), message))
machines = []
for instance in instances:
if instance.instance_state not in ("running", "pending"):
continue
if group_name not in instance.reservation.groups:
continue
machines.append(machine_from_instance(instance))
if instance_ids:
# We were asked for a specific list of machines, and if we can't
# completely fulfil that request we should blow up.
found_instance_ids = set(m.instance_id for m in machines)
missing = set(instance_ids) - found_instance_ids
if missing:
raise MachinesNotFound(missing)
returnValue(machines)
@inlineCallbacks
def destroy_environment(self):
"""Terminate all associated machines and security groups.
The super defintion of this method terminates each machine in
the environment; this needs to be augmented here by also
removing the security group for the environment.
:rtype: :class:`twisted.internet.defer.Deferred`
"""
try:
killed_machines = yield super(MachineProvider, self).\
destroy_environment()
returnValue(killed_machines)
finally:
yield destroy_environment_security_group(self)
@inlineCallbacks
def shutdown_machines(self, machines):
"""Terminate machines associated with this provider.
:param machines: machines to shut down
:type machines: list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
:return: list of terminated
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
"""
if not machines:
returnValue([])
for machine in machines:
if not isinstance(machine, EC2ProviderMachine):
raise ProviderError("Can only shut down EC2ProviderMachines; "
"got a %r" % type(machine))
ids = [m.instance_id for m in machines]
killable_machines = yield self.get_machines(ids)
if not killable_machines:
returnValue([]) # Nothing to do
killable_ids = [m.instance_id for m in killable_machines]
terminated = yield self.ec2.terminate_instances(*killable_ids)
# Pass on what was actually terminated, in the case the
# machine has somehow disappeared since get_machines
# above. This is to avoid getting EC2Error: Error Message:
# Invalid id when running ec2.describe_instances in
# remove_security_groups
terminated_ids = [info[0] for info in terminated]
yield remove_security_groups(self, terminated_ids)
returnValue(killable_machines)
def open_port(self, machine, machine_id, port, protocol="tcp"):
"""Authorizes `port` using `protocol` on EC2 for `machine`."""
return open_provider_port(self, machine, machine_id, port, protocol)
def close_port(self, machine, machine_id, port, protocol="tcp"):
"""Revokes `port` using `protocol` on EC2 for `machine`."""
return close_provider_port(self, machine, machine_id, port, protocol)
def get_opened_ports(self, machine, machine_id):
"""Returns a set of open (port, proto) pairs for `machine`."""
return get_provider_opened_ports(self, machine, machine_id)
class MachineProvider(MachineProviderBase):
"""MachineProvider for use in an EC2/S3 environment"""
def __init__(self, environment_name, config):
super(MachineProvider, self).__init__(environment_name, config)
if not config.get("ec2-uri"):
ec2_uri = get_region_uri(config.get("region", DEFAULT_REGION))
else:
ec2_uri = config.get("ec2-uri")
self._service = AWSServiceRegion(
access_key=config.get("access-key", ""),
secret_key=config.get("secret-key", ""),
ec2_uri=ec2_uri,
s3_uri=config.get("s3-uri", ""))
ssl_verify = self.config.get("ssl-hostname-verification", False)
if ssl and ssl_verify:
self._service.ec2_endpoint.ssl_hostname_verification = True
self._service.s3_endpoint.ssl_hostname_verification = True
elif ssl:
log.warn('ssl-hostname-verification is disabled for this environment')
else:
log.warn('txaws.client.ssl unavailable for SSL hostname verification')
ssl_verify = False
for endpoint, endpoint_type in [(self._service.ec2_endpoint,'EC2'),
(self._service.s3_endpoint,'S3')]:
if endpoint.scheme != 'https':
log.warn('%s API calls not using secure transport' % endpoint_type)
elif not ssl_verify:
log.warn('%s API calls encrypted but not authenticated' % endpoint_type)
if not ssl_verify:
log.warn('Ubuntu Cloud Image lookups encrypted but not authenticated')
self.s3 = self._service.get_s3_client()
self.ec2 = self._service.get_ec2_client()
@property
def provider_type(self):
return "ec2"
@property
def using_amazon(self):
return "ec2-uri" not in self.config
@inlineCallbacks
def get_constraint_set(self):
"""Return the set of constraints that are valid for this provider."""
cs = yield super(MachineProvider, self).get_constraint_set()
if 1: # These keys still need to be valid (instance-type and ec2-zone)
#if self.using_amazon:
# Expose EC2 instance types/zones on AWS itelf, not private clouds.
cs.register_generics(INSTANCE_TYPES.keys())
cs.register("ec2-zone", converter=convert_zone)
returnValue(cs)
def get_legacy_config_keys(self):
"""Return any deprecated config keys that are set"""
legacy = super(MachineProvider, self).get_legacy_config_keys()
if self.using_amazon:
# In the absence of a generic instance-type/image-id mechanism,
# these keys remain valid on private clouds.
amazon_legacy = set(("default-image-id", "default-instance-type"))
legacy.update(amazon_legacy.intersection(self.config))
return legacy
def get_serialization_data(self):
"""Get provider configuration suitable for serialization.
Also extracts credential information from the environment.
"""
data = super(MachineProvider, self).get_serialization_data()
data.setdefault("access-key", os.environ.get("AWS_ACCESS_KEY_ID"))
data.setdefault("secret-key", os.environ.get("AWS_SECRET_ACCESS_KEY"))
return data
def get_file_storage(self):
"""Retrieve an S3-backed :class:`FileStorage`."""
return FileStorage(self.s3, self.config["control-bucket"])
def start_machine(self, machine_data, master=False):
"""Start an EC2 machine.
:param dict machine_data: desired characteristics of the new machine;
it must include a "machine-id" key, and may include a "constraints"
key to specify the underlying OS and hardware.
:param bool master: if True, machine will initialize the juju admin
and run a provisioning agent, in addition to running a machine
agent.
"""
return EC2LaunchMachine.launch(self, machine_data, master)
@inlineCallbacks
def get_machines(self, instance_ids=()):
"""List machines running in the provider.
:param list instance_ids: ids of instances you want to get. Leave empty
to list every
:class:`juju.providers.ec2.machine.EC2ProviderMachine` owned by
this provider.
:return: a list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
:raises: :exc:`juju.errors.MachinesNotFound`
"""
group_name = "juju-%s" % self.environment_name
try:
instances = yield self.ec2.describe_instances(*instance_ids)
except EC2Error as error:
code = error.get_error_codes()
message = error.get_error_messages()
if code == "InvalidInstanceID.NotFound":
message = error.get_error_messages()
raise MachinesNotFound(
re.findall(r"\bi-[0-9a-f]{3,15}\b", message))
raise ProviderInteractionError(
"Unexpected EC2Error getting machines %s: %s"
% (", ".join(instance_ids), message))
machines = []
for instance in instances:
if instance.instance_state not in ("running", "pending"):
continue
if group_name not in instance.reservation.groups:
continue
machines.append(machine_from_instance(instance))
if instance_ids:
# We were asked for a specific list of machines, and if we can't
# completely fulfil that request we should blow up.
found_instance_ids = set(m.instance_id for m in machines)
missing = set(instance_ids) - found_instance_ids
if missing:
raise MachinesNotFound(missing)
returnValue(machines)
@inlineCallbacks
def destroy_environment(self):
"""Terminate all associated machines and security groups.
The super defintion of this method terminates each machine in
the environment; this needs to be augmented here by also
removing the security group for the environment.
:rtype: :class:`twisted.internet.defer.Deferred`
"""
try:
killed_machines = yield super(MachineProvider, self).\
destroy_environment()
returnValue(killed_machines)
finally:
yield destroy_environment_security_group(self)
@inlineCallbacks
def shutdown_machines(self, machines):
"""Terminate machines associated with this provider.
:param machines: machines to shut down
:type machines: list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
:return: list of terminated
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
"""
if not machines:
returnValue([])
for machine in machines:
if not isinstance(machine, EC2ProviderMachine):
raise ProviderError("Can only shut down EC2ProviderMachines; "
"got a %r" % type(machine))
ids = [m.instance_id for m in machines]
killable_machines = yield self.get_machines(ids)
if not killable_machines:
returnValue([]) # Nothing to do
killable_ids = [m.instance_id for m in killable_machines]
terminated = yield self.ec2.terminate_instances(*killable_ids)
# Pass on what was actually terminated, in the case the
# machine has somehow disappeared since get_machines
# above. This is to avoid getting EC2Error: Error Message:
# Invalid id when running ec2.describe_instances in
# remove_security_groups
terminated_ids = [info[0] for info in terminated]
yield remove_security_groups(self, terminated_ids)
returnValue(killable_machines)
def open_port(self, machine, machine_id, port, protocol="tcp"):
"""Authorizes `port` using `protocol` on EC2 for `machine`."""
return open_provider_port(self, machine, machine_id, port, protocol)
def close_port(self, machine, machine_id, port, protocol="tcp"):
"""Revokes `port` using `protocol` on EC2 for `machine`."""
return close_provider_port(self, machine, machine_id, port, protocol)
def get_opened_ports(self, machine, machine_id):
"""Returns a set of open (port, proto) pairs for `machine`."""
return get_provider_opened_ports(self, machine, machine_id)
def setUp(self):
region = AWSServiceRegion()
self.ec2 = region.get_ec2_client()
self.s3 = region.get_s3_client()
def setUp(self):
region = AWSServiceRegion()
self.ec2 = region.get_ec2_client()
self.s3 = region.get_s3_client()
