def _getClient(self):
"""
Build a txAWS S3 client using our stored credentials.
"""
creds = AWSCredentials(access_key=self.accessKey.encode('utf-8'),
secret_key=self.secretKey.encode('utf-8'))
region = AWSServiceRegion(creds=creds)
return region.get_s3_client()
def _getClient(self):
"""
Build a txAWS S3 client using our stored credentials.
"""
creds = AWSCredentials(
access_key=self.accessKey.encode('utf-8'),
secret_key=self.secretKey.encode('utf-8'))
region = AWSServiceRegion(creds=creds)
return region.get_s3_client()
def __init__(self, engine):
self.engine = engine
self.s3conn = boto.connect_s3(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
if not settings.USE_BOTO:
self.txs3conn = AWSServiceRegion(
access_key=settings.AWS_ACCESS_KEY_ID,
secret_key=settings.AWS_SECRET_ACCESS_KEY,
s3_uri=S3_US[0]['endpoint'],
# s3_uri='https://s3.amazonaws.com',
).get_s3_client()
self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE)
def __init__(self, environment_name, config):
super(MachineProvider, self).__init__(environment_name, config)
if not config.get("ec2-uri"):
ec2_uri = get_region_uri(config.get("region", "us-east-1"))
else:
ec2_uri = config.get("ec2-uri")
self._service = AWSServiceRegion(
access_key=config.get("access-key", ""),
secret_key=config.get("secret-key", ""),
ec2_uri=ec2_uri,
s3_uri=config.get("s3-uri", ""))
self.s3 = self._service.get_s3_client()
self.ec2 = self._service.get_ec2_client()
def main(reactor):
aws = AWSServiceRegion(AWSCredentials())
agent = Agent(reactor)
route53 = get_route53_client(agent, aws)
d = route53.list_resource_record_sets(zone_id="Z2T2TSJ409GHZ9")
d.addCallback(pprint)
return d
def __init__(self, environment_name, config):
super(MachineProvider, self).__init__(environment_name, config)
if not config.get("ec2-uri"):
ec2_uri = get_region_uri(config.get("region", DEFAULT_REGION))
else:
ec2_uri = config.get("ec2-uri")
self._service = AWSServiceRegion(
access_key=config.get("access-key", ""),
secret_key=config.get("secret-key", ""),
ec2_uri=ec2_uri,
s3_uri=config.get("s3-uri", ""))
ssl_verify = self.config.get("ssl-hostname-verification", False)
if ssl and ssl_verify:
self._service.ec2_endpoint.ssl_hostname_verification = True
self._service.s3_endpoint.ssl_hostname_verification = True
elif ssl:
log.warn('ssl-hostname-verification is disabled for this environment')
else:
log.warn('txaws.client.ssl unavailable for SSL hostname verification')
ssl_verify = False
for endpoint, endpoint_type in [(self._service.ec2_endpoint,'EC2'),
(self._service.s3_endpoint,'S3')]:
if endpoint.scheme != 'https':
log.warn('%s API calls not using secure transport' % endpoint_type)
elif not ssl_verify:
log.warn('%s API calls encrypted but not authenticated' % endpoint_type)
if not ssl_verify:
log.warn('Ubuntu Cloud Image lookups encrypted but not authenticated')
self.s3 = self._service.get_s3_client()
self.ec2 = self._service.get_ec2_client()
def test_error_changes(self):
duplicate_resource = POSTableData(
sample_create_resource_record_sets_error_result.xml,
b"text/xml",
BAD_REQUEST,
)
zone_id = "1234ABCDEF"
agent = RequestTraversalAgent(
static_resource({
b"2013-04-01": {
b"hostedzone": {
zone_id.encode("ascii"): {
b"rrset": duplicate_resource,
},
},
},
}))
aws = AWSServiceRegion(access_key="abc", secret_key="def")
client = get_route53_client(agent, aws, uncooperator())
err = self.failureResultOf(
client.change_resource_record_sets(
zone_id=zone_id,
changes=[
create_rrset(
sample_create_resource_record_sets_error_result.rrset)
],
), Route53Error)
expected = {
'Code': 'InvalidChangeBatch',
'Message':
"[Tried to create resource record set [name='duplicate.example.invalid.', type='CNAME'] but it already exists]",
'Type': 'Sender',
}
self.assertEqual(err.value.errors, [expected])
def test_creation_with_keys_and_creds(self):
"""
creds take precedence over individual access key/secret key pairs.
"""
region = AWSServiceRegion(self.creds, access_key="baz",
secret_key="quux")
self.assertEquals(region.creds.access_key, "foo")
self.assertEquals(region.creds.secret_key, "bar")
def __init__(self, engine):
self.engine = engine
self.s3conn = boto.connect_s3(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
self.txs3conn = AWSServiceRegion(
access_key=settings.AWS_ACCESS_KEY_ID,
secret_key=settings.AWS_SECRET_ACCESS_KEY,
s3_uri=S3_EU_WEST[0]['endpoint'],
).get_s3_client()
self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE)
def __init__(self, engine):
self.engine = engine
self.s3conn = boto.connect_s3(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
if not settings.USE_BOTO:
self.txs3conn = AWSServiceRegion(
access_key=settings.AWS_ACCESS_KEY_ID,
secret_key=settings.AWS_SECRET_ACCESS_KEY,
s3_uri=settings.AWS_S3_ENDPOINT,
).get_s3_client()
self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE)
def __init__(self, reactor):
from txaws.service import AWSServiceRegion
gtk.StatusIcon.__init__(self)
self.set_from_stock(gtk.STOCK_NETWORK)
self.set_visible(True)
self.reactor = reactor
self.connect("activate", self.on_activate)
self.probing = False
# Nested import because otherwise we get "reactor already installed".
self.password_dialog = None
try:
creds = AWSCredentials()
except ValueError:
creds = self.from_gnomekeyring()
self.region = AWSServiceRegion(creds)
self.create_client(creds)
menu = """
<ui>
<menubar name="Menubar">
<menu action="Menu">
<menuitem action="Stop instances"/>
</menu>
</menubar>
</ui>
"""
actions = [
("Menu", None, "Menu"),
("Stop instances", gtk.STOCK_STOP, "_Stop instances...", None,
"Stop instances", self.on_stop_instances),
]
ag = gtk.ActionGroup("Actions")
ag.add_actions(actions)
self.manager = gtk.UIManager()
self.manager.insert_action_group(ag, 0)
self.manager.add_ui_from_string(menu)
self.menu = self.manager.get_widget(
"/Menubar/Menu/Stop instances").props.parent
self.connect("popup-menu", self.on_popup_menu)
def __init__(self, engine):
self.engine = engine
self.s3conn = boto3.Session(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name=settings.AWS_REGION
)
if not settings.USE_BOTO:
self.txs3conn = AWSServiceRegion(
access_key=settings.AWS_ACCESS_KEY_ID,
secret_key=settings.AWS_SECRET_ACCESS_KEY,
s3_uri=S3_EU_WEST[0]['endpoint'],
).get_s3_client()
self.s3resource = self.s3conn.resource('s3')
def test_some_zones(self):
agent = RequestTraversalAgent(
static_resource({
b"2013-04-01": {
b"hostedzone":
Data(
sample_list_hosted_zones_result.xml,
b"text/xml",
),
},
}))
aws = AWSServiceRegion(access_key="abc", secret_key="def")
client = get_route53_client(agent, aws, uncooperator())
zones = self.successResultOf(client.list_hosted_zones())
expected = [HostedZone(**sample_list_hosted_zones_result.details)]
self.assertEquals(expected, zones)
def _client_for_rrsets(self, zone_id, rrsets_xml):
agent = RequestTraversalAgent(
static_resource({
b"2013-04-01": {
b"hostedzone": {
zone_id: {
b"rrset": Data(
rrsets_xml,
b"text/xml",
)
}
}
}
}))
aws = AWSServiceRegion(access_key="abc", secret_key="def")
return get_route53_client(agent, aws, uncooperator())
def _finish_convergence_service(k8s_client, options, subscription_client):
k8s = KubeClient(k8s=k8s_client)
access_key_id = FilePath(
options["aws-access-key-id-path"]).getContent().strip()
secret_access_key = FilePath(
options["aws-secret-access-key-path"]).getContent().strip()
aws = AWSServiceRegion(creds=AWSCredentials(
access_key=access_key_id,
secret_key=secret_access_key,
))
Message.log(
event=u"convergence-service:key-notification",
key_id=access_key_id.decode("ascii"),
secret_key_hash=sha256(secret_access_key).hexdigest().decode("ascii"),
)
# XXX I get to leave a ton of fields empty because I happen to know
# they're not used in this codepath. :/ Maybe this suggests something has
# gone wrong ...
config = DeploymentConfiguration(
domain=options["domain"].decode("ascii"),
kubernetes_namespace=options["kubernetes-namespace"].decode("ascii"),
subscription_manager_endpoint=URL.fromText(
options["endpoint"].decode("ascii")),
s3_access_key_id=access_key_id.decode("ascii"),
s3_secret_key=secret_access_key.decode("ascii"),
introducer_image=options["introducer-image"].decode("ascii"),
storageserver_image=options["storageserver-image"].decode("ascii"),
log_gatherer_furl=None,
stats_gatherer_furl=None,
)
return TimerService(
options["interval"],
divert_errors_to_log(converge, u"subscription_converger"),
config,
subscription_client,
k8s,
aws,
)
def _finish_convergence_service(
k8s_client, options, subscription_client, reactor,
):
k8s = KubeClient(k8s=k8s_client)
access_key_id = FilePath(options["aws-access-key-id-path"]).getContent().strip()
secret_access_key = FilePath(options["aws-secret-access-key-path"]).getContent().strip()
aws = AWSServiceRegion(creds=AWSCredentials(
access_key=access_key_id,
secret_key=secret_access_key,
))
Message.log(
event=u"convergence-service:key-notification",
key_id=access_key_id.decode("ascii"),
secret_key_hash=sha256(secret_access_key).hexdigest().decode("ascii"),
)
config = DeploymentConfiguration(
domain=options["domain"].decode("ascii"),
kubernetes_namespace=options["kubernetes-namespace"].decode("ascii"),
subscription_manager_endpoint=URL.fromText(options["endpoint"].decode("ascii")),
s3_access_key_id=access_key_id.decode("ascii"),
s3_secret_key=secret_access_key.decode("ascii"),
introducer_image=options["introducer-image"].decode("ascii"),
storageserver_image=options["storageserver-image"].decode("ascii"),
log_gatherer_furl=options["log-gatherer-furl"],
stats_gatherer_furl=options["stats-gatherer-furl"],
)
return _convergence_service(
reactor,
options["interval"],
config,
subscription_client,
k8s,
aws,
)
def run(self):
"""
Run the configured method and write the HTTP response status and text
to the output stream.
"""
region = AWSServiceRegion(access_key=self.key, secret_key=self.secret,
uri=self.endpoint)
query = self.query_factory(action=self.action, creds=region.creds,
endpoint=region.ec2_endpoint,
other_params=self.parameters)
def write_response(response):
print >> self.output, "URL: %s" % query.client.url
print >> self.output
print >> self.output, "HTTP status code: %s" % query.client.status
print >> self.output
print >> self.output, response
def write_error(failure):
if failure.check(AWSError):
message = failure.value.original
else:
message = failure.getErrorMessage()
if message.startswith("Error Message: "):
message = message[len("Error Message: "):]
print >> self.output, "URL: %s" % query.client.url
print >> self.output
if getattr(query.client, "status", None) is not None:
print >> self.output, "HTTP status code: %s" % (
query.client.status,)
print >> self.output
print >> self.output, message
if getattr(failure.value, "response", None) is not None:
print >> self.output
print >> self.output, failure.value.response
deferred = query.submit()
deferred.addCallback(write_response)
deferred.addErrback(write_error)
return deferred
def test_some_changes(self):
change_resource = POSTableData(
sample_change_resource_record_sets_result.xml,
b"text/xml",
)
zone_id = u"ABCDEF1234"
agent = RequestTraversalAgent(
static_resource({
b"2013-04-01": {
b"hostedzone": {
zone_id.encode("ascii"): {
b"rrset": change_resource,
}
},
},
}))
aws = AWSServiceRegion(access_key="abc", secret_key="def")
client = get_route53_client(agent, aws, uncooperator())
self.successResultOf(
client.change_resource_record_sets(
zone_id=zone_id,
changes=[
create_rrset(
sample_change_resource_record_sets_result.rrset),
delete_rrset(
sample_change_resource_record_sets_result.rrset),
upsert_rrset(
sample_change_resource_record_sets_result.rrset),
],
))
# Ack, what a pathetic assertion.
change_template = u"<Change><Action>{action}</Action><ResourceRecordSet><Name>example.invalid.</Name><Type>NS</Type><TTL>86400</TTL><ResourceRecords><ResourceRecord><Value>ns1.example.invalid.</Value></ResourceRecord><ResourceRecord><Value>ns2.example.invalid.</Value></ResourceRecord></ResourceRecords></ResourceRecordSet></Change>"
changes = [
change_template.format(action=u"CREATE"),
change_template.format(action=u"DELETE"),
change_template.format(action=u"UPSERT"),
]
expected = u"""\
<?xml version="1.0" encoding="UTF-8"?>
<ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ChangeBatch><Changes>{changes}</Changes></ChangeBatch></ChangeResourceRecordSetsRequest>""".format(
changes=u"".join(changes)).encode("utf-8")
self.assertEqual((expected, ), change_resource.posted)
class AWSStatusIcon(gtk.StatusIcon):
"""A status icon shown when instances are running."""
def __init__(self, reactor):
from txaws.service import AWSServiceRegion
gtk.StatusIcon.__init__(self)
self.set_from_stock(gtk.STOCK_NETWORK)
self.set_visible(True)
self.reactor = reactor
self.connect("activate", self.on_activate)
self.probing = False
# Nested import because otherwise we get "reactor already installed".
self.password_dialog = None
try:
creds = AWSCredentials()
except ValueError:
creds = self.from_gnomekeyring()
self.region = AWSServiceRegion(creds)
self.create_client(creds)
menu = """
<ui>
<menubar name="Menubar">
<menu action="Menu">
<menuitem action="Stop instances"/>
</menu>
</menubar>
</ui>
"""
actions = [
("Menu", None, "Menu"),
("Stop instances", gtk.STOCK_STOP, "_Stop instances...", None,
"Stop instances", self.on_stop_instances),
]
ag = gtk.ActionGroup("Actions")
ag.add_actions(actions)
self.manager = gtk.UIManager()
self.manager.insert_action_group(ag, 0)
self.manager.add_ui_from_string(menu)
self.menu = self.manager.get_widget(
"/Menubar/Menu/Stop instances").props.parent
self.connect("popup-menu", self.on_popup_menu)
def create_client(self, creds):
if creds is not None:
self.client = self.region.get_ec2_client()
self.on_activate(None)
else:
# waiting on user entered credentials.
self.client = None
def from_gnomekeyring(self):
# Try for gtk gui specific credentials.
try:
items = gnomekeyring.find_items_sync(
gnomekeyring.ITEM_GENERIC_SECRET,
{
"aws-host": "aws.amazon.com",
})
except (gnomekeyring.NoMatchError,
gnomekeyring.DeniedError):
self.show_a_password_dialog()
return None
else:
key_id, secret_key = items[0].secret.split(":")
return AWSCredentials(access_key=key_id, secret_key=secret_key)
def show_a_password_dialog(self):
self.password_dialog = gtk.Dialog(
"Enter your AWS credentals", None, gtk.DIALOG_MODAL,
(gtk.STOCK_OK, gtk.RESPONSE_ACCEPT,
gtk.STOCK_CANCEL,
gtk.RESPONSE_REJECT))
content = self.password_dialog.get_content_area()
def add_entry(name):
box = gtk.HBox()
box.show()
content.add(box)
label = gtk.Label(name)
label.show()
box.add(label)
entry = gtk.Entry()
entry.show()
box.add(entry)
label.set_use_underline(True)
label.set_mnemonic_widget(entry)
add_entry("AWS _Access Key ID")
add_entry("AWS _Secret Key")
self.password_dialog.show()
self.password_dialog.connect("response", self.save_key)
self.password_dialog.run()
def on_activate(self, data):
if self.probing or not self.client:
# don't ask multiple times, and don't ask until we have
# credentials.
return
self.probing = True
deferred = self.client.describe_instances()
deferred.addCallbacks(self.showhide, self.describe_error)
def on_popup_menu(self, status, button, time):
self.menu.popup(None, None, None, button, time)
def on_stop_instances(self, data):
# It would be nice to popup a window to select instances.. TODO.
deferred = self.client.describe_instances()
deferred.addCallbacks(self.shutdown_instances, self.show_error)
def save_key(self, response_id, data):
try:
if data != gtk.RESPONSE_ACCEPT:
# User cancelled. They can ask for the password again somehow.
return
content = self.password_dialog.get_content_area()
key_id = content.get_children()[0].get_children()[1].get_text()
secret_key = content.get_children()[1].get_children()[1].get_text()
creds = AWSCredentials(access_key=key_id, secret_key=secret_key)
self.create_client(creds)
gnomekeyring.item_create_sync(
None,
gnomekeyring.ITEM_GENERIC_SECRET,
"AWS access credentials",
{"aws-host": "aws.amazon.com"},
"%s:%s" % (key_id, secret_key), True)
finally:
self.password_dialog.hide()
# XXX? Does this leak?
self.password_dialog = None
def showhide(self, reservation):
active = 0
for instance in reservation:
if instance.instance_state == "running":
active += 1
self.set_tooltip("AWS Status - %d instances" % active)
self.set_visible(active != 0)
self.queue_check()
def shutdown_instances(self, reservation):
d = self.client.terminate_instances(
*[instance.instance_id for instance in reservation])
d.addCallbacks(self.on_activate, self.show_error)
def queue_check(self):
self.probing = False
self.reactor.callLater(60, self.on_activate, None)
def show_error(self, error):
# debugging output for now.
print error.value
try:
print error.value.response
except:
pass
def describe_error(self, error):
from twisted.internet.defer import TimeoutError
if isinstance(error.value, TimeoutError):
# timeout errors can be ignored - transient network issue or some
# such.
pass
else:
# debugging output for now.
self.show_error(error)
self.queue_check()
def setUp(self):
region = AWSServiceRegion()
self.ec2 = region.get_ec2_client()
self.s3 = region.get_s3_client()
def setUp(self):
self.creds = AWSCredentials("foo", "bar")
self.region = AWSServiceRegion(creds=self.creds)
def setUp(self):
region = AWSServiceRegion()
self.ec2 = region.get_ec2_client()
self.s3 = region.get_s3_client()
class MachineProvider(MachineProviderBase):
"""MachineProvider for use in an EC2/S3 environment"""
def __init__(self, environment_name, config):
super(MachineProvider, self).__init__(environment_name, config)
if not config.get("ec2-uri"):
ec2_uri = get_region_uri(config.get("region", DEFAULT_REGION))
else:
ec2_uri = config.get("ec2-uri")
self._service = AWSServiceRegion(
access_key=config.get("access-key", ""),
secret_key=config.get("secret-key", ""),
ec2_uri=ec2_uri,
s3_uri=config.get("s3-uri", ""))
ssl_verify = self.config.get("ssl-hostname-verification", False)
if ssl and ssl_verify:
self._service.ec2_endpoint.ssl_hostname_verification = True
self._service.s3_endpoint.ssl_hostname_verification = True
elif ssl:
log.warn('ssl-hostname-verification is disabled for this environment')
else:
log.warn('txaws.client.ssl unavailable for SSL hostname verification')
ssl_verify = False
for endpoint, endpoint_type in [(self._service.ec2_endpoint,'EC2'),
(self._service.s3_endpoint,'S3')]:
if endpoint.scheme != 'https':
log.warn('%s API calls not using secure transport' % endpoint_type)
elif not ssl_verify:
log.warn('%s API calls encrypted but not authenticated' % endpoint_type)
if not ssl_verify:
log.warn('Ubuntu Cloud Image lookups encrypted but not authenticated')
self.s3 = self._service.get_s3_client()
self.ec2 = self._service.get_ec2_client()
@property
def provider_type(self):
return "ec2"
@property
def using_amazon(self):
return "ec2-uri" not in self.config
@inlineCallbacks
def get_constraint_set(self):
"""Return the set of constraints that are valid for this provider."""
cs = yield super(MachineProvider, self).get_constraint_set()
if 1: # These keys still need to be valid (instance-type and ec2-zone)
#if self.using_amazon:
# Expose EC2 instance types/zones on AWS itelf, not private clouds.
cs.register_generics(INSTANCE_TYPES.keys())
cs.register("ec2-zone", converter=convert_zone)
returnValue(cs)
def get_legacy_config_keys(self):
"""Return any deprecated config keys that are set"""
legacy = super(MachineProvider, self).get_legacy_config_keys()
if self.using_amazon:
# In the absence of a generic instance-type/image-id mechanism,
# these keys remain valid on private clouds.
amazon_legacy = set(("default-image-id", "default-instance-type"))
legacy.update(amazon_legacy.intersection(self.config))
return legacy
def get_serialization_data(self):
"""Get provider configuration suitable for serialization.
Also extracts credential information from the environment.
"""
data = super(MachineProvider, self).get_serialization_data()
data.setdefault("access-key", os.environ.get("AWS_ACCESS_KEY_ID"))
data.setdefault("secret-key", os.environ.get("AWS_SECRET_ACCESS_KEY"))
return data
def get_file_storage(self):
"""Retrieve an S3-backed :class:`FileStorage`."""
return FileStorage(self.s3, self.config["control-bucket"])
def start_machine(self, machine_data, master=False):
"""Start an EC2 machine.
:param dict machine_data: desired characteristics of the new machine;
it must include a "machine-id" key, and may include a "constraints"
key to specify the underlying OS and hardware.
:param bool master: if True, machine will initialize the juju admin
and run a provisioning agent, in addition to running a machine
agent.
"""
return EC2LaunchMachine.launch(self, machine_data, master)
@inlineCallbacks
def get_machines(self, instance_ids=()):
"""List machines running in the provider.
:param list instance_ids: ids of instances you want to get. Leave empty
to list every
:class:`juju.providers.ec2.machine.EC2ProviderMachine` owned by
this provider.
:return: a list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
:raises: :exc:`juju.errors.MachinesNotFound`
"""
group_name = "juju-%s" % self.environment_name
try:
instances = yield self.ec2.describe_instances(*instance_ids)
except EC2Error as error:
code = error.get_error_codes()
message = error.get_error_messages()
if code == "InvalidInstanceID.NotFound":
message = error.get_error_messages()
raise MachinesNotFound(
re.findall(r"\bi-[0-9a-f]{3,15}\b", message))
raise ProviderInteractionError(
"Unexpected EC2Error getting machines %s: %s"
% (", ".join(instance_ids), message))
machines = []
for instance in instances:
if instance.instance_state not in ("running", "pending"):
continue
if group_name not in instance.reservation.groups:
continue
machines.append(machine_from_instance(instance))
if instance_ids:
# We were asked for a specific list of machines, and if we can't
# completely fulfil that request we should blow up.
found_instance_ids = set(m.instance_id for m in machines)
missing = set(instance_ids) - found_instance_ids
if missing:
raise MachinesNotFound(missing)
returnValue(machines)
@inlineCallbacks
def destroy_environment(self):
"""Terminate all associated machines and security groups.
The super defintion of this method terminates each machine in
the environment; this needs to be augmented here by also
removing the security group for the environment.
:rtype: :class:`twisted.internet.defer.Deferred`
"""
try:
killed_machines = yield super(MachineProvider, self).\
destroy_environment()
returnValue(killed_machines)
finally:
yield destroy_environment_security_group(self)
@inlineCallbacks
def shutdown_machines(self, machines):
"""Terminate machines associated with this provider.
:param machines: machines to shut down
:type machines: list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
:return: list of terminated
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
"""
if not machines:
returnValue([])
for machine in machines:
if not isinstance(machine, EC2ProviderMachine):
raise ProviderError("Can only shut down EC2ProviderMachines; "
"got a %r" % type(machine))
ids = [m.instance_id for m in machines]
killable_machines = yield self.get_machines(ids)
if not killable_machines:
returnValue([]) # Nothing to do
killable_ids = [m.instance_id for m in killable_machines]
terminated = yield self.ec2.terminate_instances(*killable_ids)
# Pass on what was actually terminated, in the case the
# machine has somehow disappeared since get_machines
# above. This is to avoid getting EC2Error: Error Message:
# Invalid id when running ec2.describe_instances in
# remove_security_groups
terminated_ids = [info[0] for info in terminated]
yield remove_security_groups(self, terminated_ids)
returnValue(killable_machines)
def open_port(self, machine, machine_id, port, protocol="tcp"):
"""Authorizes `port` using `protocol` on EC2 for `machine`."""
return open_provider_port(self, machine, machine_id, port, protocol)
def close_port(self, machine, machine_id, port, protocol="tcp"):
"""Revokes `port` using `protocol` on EC2 for `machine`."""
return close_provider_port(self, machine, machine_id, port, protocol)
def get_opened_ports(self, machine, machine_id):
"""Returns a set of open (port, proto) pairs for `machine`."""
return get_provider_opened_ports(self, machine, machine_id)
class S3Downloader(object):
""" Pipeline process which downloads a media file from S3 """
def __init__(self, engine):
self.engine = engine
self.s3conn = boto.connect_s3(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
if not settings.USE_BOTO:
self.txs3conn = AWSServiceRegion(
access_key=settings.AWS_ACCESS_KEY_ID,
secret_key=settings.AWS_SECRET_ACCESS_KEY,
s3_uri=S3_US[0]['endpoint'],
# s3_uri='https://s3.amazonaws.com',
).get_s3_client()
self.botobucket = self.s3conn.get_bucket(settings.IMAGES_STORE)
@defer.inlineCallbacks
def _get_data_from_s3_tx(self, path):
""" txAWS GET from S3 """
image = yield self.txs3conn.get_object(
settings.IMAGES_STORE,
str(path),
)
defer.returnValue(image)
def _get_data_from_s3(self, path):
""" boto GET from S3 """
key = self.botobucket.get_key(path)
data = key.get_contents_as_string()
return data
@time_on_statsd(statsd_name(), 's3_downloader')
def process_image(self, payload, **kwargs):
""" Gets image data from S3.
This attempts to download from s3 settings.ATTEMPTS times and timeouts after
settings.S3_TIMEOUT """
def _create_deferred(timeout=0):
""" Creates a deferred which will run after a given delay """
if settings.USE_BOTO:
dfd = task.deferLater(
reactor, timeout,
threads.deferToThread, self._get_data_from_s3, payload['image_path']
)
return dfd
else:
dfd = task.deferLater(
reactor, timeout, self._get_data_from_s3_tx, payload['image_path']
)
return dfd
def _s3callback(deferred_list_result):
""" When one of the requests has completed, cancel the rest """
[dfd.cancel() for dfd in dfds_list if not dfd.called]
if not deferred_list_result[0]:
raise NoDataInS3Error()
payload['original_image'] = deferred_list_result[0]
return payload
def _timeout_and_fail(dfds_list):
""" If none of the defers has finished by (attempts+1)*timeout then
cancel and return an error """
[dfd.cancel() for dfd in dfds_list if not dfd.called]
def _surpress_cancel_error(result):
if isinstance(result, defer.CancelledError):
pass
# Skip if already exists from cache
if 'original_image' in payload.keys():
return payload
if settings.DEBUG:
log.msg(
"[%s] Starting S3 Download" % datetime.now().isoformat(), logLevel=logging.DEBUG
)
# Make a deferred list of download attempts that have their predefined starting
# times baked into the deferred. Return when any deferred has a successful result
# Keep a list of the original deferred as we cannot access them once in DeferredList
dfds_list = []
for attempt in range(0, settings.S3_ATTEMPTS):
dfds_list.append(_create_deferred(timeout=attempt*settings.S3_TIMEOUT))
dfds_list[-1].addErrback(_surpress_cancel_error)
dfds = defer.DeferredList(dfds_list, fireOnOneCallback=True)
dfds.addCallback(_s3callback)
# Auto cancel requests which don't fire after their max timeout
reactor.callLater(
settings.S3_ATTEMPTS*settings.S3_TIMEOUT, _timeout_and_fail, dfds_list
)
return dfds
class MachineProvider(MachineProviderBase):
"""MachineProvider for use in an EC2/S3 environment"""
def __init__(self, environment_name, config):
super(MachineProvider, self).__init__(environment_name, config)
if not config.get("ec2-uri"):
ec2_uri = get_region_uri(config.get("region", "us-east-1"))
else:
ec2_uri = config.get("ec2-uri")
self._service = AWSServiceRegion(
access_key=config.get("access-key", ""),
secret_key=config.get("secret-key", ""),
ec2_uri=ec2_uri,
s3_uri=config.get("s3-uri", ""))
self.s3 = self._service.get_s3_client()
self.ec2 = self._service.get_ec2_client()
@property
def provider_type(self):
return "ec2"
def get_serialization_data(self):
"""Get provider configuration suitable for serialization.
Also extracts credential information from the environment.
"""
data = super(MachineProvider, self).get_serialization_data()
data.setdefault("access-key", os.environ.get("AWS_ACCESS_KEY_ID"))
data.setdefault("secret-key", os.environ.get("AWS_SECRET_ACCESS_KEY"))
return data
def get_file_storage(self):
"""Retrieve an S3-backed :class:`FileStorage`."""
return FileStorage(self.s3, self.config["control-bucket"])
def start_machine(self, machine_data, master=False):
"""Start an EC2 machine.
:param dict machine_data: desired characteristics of the new machine;
it must include a "machine-id" key, and may include a "constraints"
key to specify the underlying OS and hardware.
:param bool master: if True, machine will initialize the juju admin
and run a provisioning agent, in addition to running a machine
agent.
"""
if "machine-id" not in machine_data:
return fail(ProviderError(
"Cannot launch a machine without specifying a machine-id"))
machine_id = machine_data["machine-id"]
constraints = machine_data.get("constraints", {})
return EC2LaunchMachine(self, master, constraints).run(machine_id)
@inlineCallbacks
def get_machines(self, instance_ids=()):
"""List machines running in the provider.
:param list instance_ids: ids of instances you want to get. Leave empty
to list every
:class:`juju.providers.ec2.machine.EC2ProviderMachine` owned by
this provider.
:return: a list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
:raises: :exc:`juju.errors.MachinesNotFound`
"""
group_name = "juju-%s" % self.environment_name
try:
instances = yield self.ec2.describe_instances(*instance_ids)
except EC2Error as error:
code = error.get_error_codes()
message = error.get_error_messages()
if code == "InvalidInstanceID.NotFound":
message = error.get_error_messages()
raise MachinesNotFound(
re.findall(r"\bi-[0-9a-f]{3,15}\b", message))
raise ProviderInteractionError(
"Unexpected EC2Error getting machines %s: %s"
% (", ".join(instance_ids), message))
machines = []
for instance in instances:
if instance.instance_state not in ("running", "pending"):
continue
if group_name not in instance.reservation.groups:
continue
machines.append(machine_from_instance(instance))
if instance_ids:
# We were asked for a specific list of machines, and if we can't
# completely fulfil that request we should blow up.
found_instance_ids = set(m.instance_id for m in machines)
missing = set(instance_ids) - found_instance_ids
if missing:
raise MachinesNotFound(missing)
returnValue(machines)
@inlineCallbacks
def destroy_environment(self):
"""Terminate all associated machines and security groups.
The super defintion of this method terminates each machine in
the environment; this needs to be augmented here by also
removing the security group for the environment.
:rtype: :class:`twisted.internet.defer.Deferred`
"""
try:
killed_machines = yield super(MachineProvider, self).\
destroy_environment()
returnValue(killed_machines)
finally:
yield destroy_environment_security_group(self)
@inlineCallbacks
def shutdown_machines(self, machines):
"""Terminate machines associated with this provider.
:param machines: machines to shut down
:type machines: list of
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
:return: list of terminated
:class:`juju.providers.ec2.machine.EC2ProviderMachine`
instances
:rtype: :class:`twisted.internet.defer.Deferred`
"""
if not machines:
returnValue([])
for machine in machines:
if not isinstance(machine, EC2ProviderMachine):
raise ProviderError("Can only shut down EC2ProviderMachines; "
"got a %r" % type(machine))
ids = [m.instance_id for m in machines]
killable_machines = yield self.get_machines(ids)
if not killable_machines:
returnValue([]) # Nothing to do
killable_ids = [m.instance_id for m in killable_machines]
terminated = yield self.ec2.terminate_instances(*killable_ids)
# Pass on what was actually terminated, in the case the
# machine has somehow disappeared since get_machines
# above. This is to avoid getting EC2Error: Error Message:
# Invalid id when running ec2.describe_instances in
# remove_security_groups
terminated_ids = [info[0] for info in terminated]
yield remove_security_groups(self, terminated_ids)
returnValue(killable_machines)
def open_port(self, machine, machine_id, port, protocol="tcp"):
"""Authorizes `port` using `protocol` on EC2 for `machine`."""
return open_provider_port(self, machine, machine_id, port, protocol)
def close_port(self, machine, machine_id, port, protocol="tcp"):
"""Revokes `port` using `protocol` on EC2 for `machine`."""
return close_provider_port(self, machine, machine_id, port, protocol)
def get_opened_ports(self, machine, machine_id):
"""Returns a set of open (port, proto) pairs for `machine`."""
return get_provider_opened_ports(self, machine, machine_id)
class AWSServiceRegionTestCase(TestCase):
def setUp(self):
self.creds = AWSCredentials("foo", "bar")
self.region = AWSServiceRegion(creds=self.creds)
def test_simple_creation(self):
self.assertEquals(self.creds, self.region.creds)
self.assertEquals(self.region._clients, {})
self.assertEquals(self.region.ec2_endpoint.get_uri(), EC2_ENDPOINT_US)
def test_creation_with_keys(self):
region = AWSServiceRegion(access_key="baz", secret_key="quux")
self.assertEquals(region.creds.access_key, "baz")
self.assertEquals(region.creds.secret_key, "quux")
def test_creation_with_keys_and_creds(self):
"""
creds take precedence over individual access key/secret key pairs.
"""
region = AWSServiceRegion(self.creds, access_key="baz",
secret_key="quux")
self.assertEquals(region.creds.access_key, "foo")
self.assertEquals(region.creds.secret_key, "bar")
def test_creation_with_uri(self):
region = AWSServiceRegion(
creds=self.creds, ec2_uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_uri_backwards_compatible(self):
region = AWSServiceRegion(
creds=self.creds, uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_uri_and_region(self):
region = AWSServiceRegion(
creds=self.creds, region=REGION_EU, ec2_uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_region_override(self):
region = AWSServiceRegion(creds=self.creds, region=REGION_EU)
self.assertEquals(region.ec2_endpoint.get_uri(), EC2_ENDPOINT_EU)
def test_get_ec2_client_with_empty_cache(self):
key = str(EC2Client) + str(self.creds) + str(self.region.ec2_endpoint)
original_client = self.region._clients.get(key)
new_client = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint)
self.assertEquals(original_client, None)
self.assertTrue(isinstance(new_client, EC2Client))
self.assertNotEquals(original_client, new_client)
def test_get_ec2_client_from_cache_default(self):
client1 = self.region.get_ec2_client()
client2 = self.region.get_ec2_client()
self.assertTrue(isinstance(client1, EC2Client))
self.assertTrue(isinstance(client2, EC2Client))
self.assertEquals(client1, client2)
def test_get_ec2_client_from_cache(self):
client1 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint)
client2 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint)
self.assertTrue(isinstance(client1, EC2Client))
self.assertTrue(isinstance(client2, EC2Client))
self.assertEquals(client1, client2)
def test_get_ec2_client_from_cache_with_purge(self):
client1 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint,
purge_cache=True)
client2 = self.region.get_client(
EC2Client, creds=self.creds, endpoint=self.region.ec2_endpoint,
purge_cache=True)
self.assertTrue(isinstance(client1, EC2Client))
self.assertTrue(isinstance(client2, EC2Client))
self.assertNotEquals(client1, client2)
def test_get_s3_client_with_empty_cache(self):
key = str(S3Client) + str(self.creds) + str(self.region.s3_endpoint)
original_client = self.region._clients.get(key)
new_client = self.region.get_client(
S3Client, creds=self.creds, endpoint=self.region.s3_endpoint)
self.assertEquals(original_client, None)
self.assertTrue(isinstance(new_client, S3Client))
self.assertNotEquals(original_client, new_client)
test_get_s3_client_with_empty_cache.skip = s3clientSkip
def test_creation_with_uri_and_region(self):
region = AWSServiceRegion(
creds=self.creds, region=REGION_EU, ec2_uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
def test_creation_with_keys(self):
region = AWSServiceRegion(access_key="baz", secret_key="quux")
self.assertEquals(region.creds.access_key, "baz")
self.assertEquals(region.creds.secret_key, "quux")
def set_region(self, creds):
from txaws.service import AWSServiceRegion
self.region = AWSServiceRegion(creds)
def test_creation_with_uri_backwards_compatible(self):
region = AWSServiceRegion(
creds=self.creds, uri="http://foo/bar")
self.assertEquals(region.ec2_endpoint.get_uri(), "http://foo/bar")
#
# - in case they leak out of the test suite somehow
#
# - in case the implementation is broken and does something destructive
#
# - in case malicious code is inserted somehow (eg, you run
# tests on code submitted by another developer)
#
# As far as I can tell there's no way to isolate an API user
# from _some_ of the parent account's S3 buckets. Therefore,
# isolation probably involves registering a new top-level AWS
# account and dedicating it to testing purposes.
try:
access_key = environ["TXAWS_INTEGRATION_AWS_ACCESS_KEY_ID"]
secret_key = environ["TXAWS_INTEGRATION_AWS_SECRET_ACCESS_KEY"]
except KeyError as e:
case.skipTest("Missing {} environment variable.".format(e))
else:
credentials = AWSCredentials(
access_key=access_key,
secret_key=secret_key,
)
return AWSServiceRegion(credentials)
def get_memory_service(case):
return FakeAWSServiceRegion(
access_key="fake access key",
secret_key="fake secret key",
)
def test_creation_with_region_override(self):
region = AWSServiceRegion(creds=self.creds, region=REGION_EU)
self.assertEquals(region.ec2_endpoint.get_uri(), EC2_ENDPOINT_EU)
def set_region(self, creds):
from txaws.service import AWSServiceRegion
self.region = AWSServiceRegion(creds)
