def _service_is_active(self):
"""
Check that the database guest is active.
This function is meant to be called with poll_until to check that
the guest is alive before sending a 'create' message. This prevents
over billing a customer for a instance that they can never use.
Returns: boolean if the service is active.
Raises: TroveError if the service is in a failure state.
"""
service = InstanceServiceStatus.find_by(instance_id=self.id)
status = service.get_status()
if status == rd_instance.ServiceStatuses.RUNNING:
return True
elif status not in [
rd_instance.ServiceStatuses.NEW,
rd_instance.ServiceStatuses.BUILDING
]:
raise TroveError(_("Service not active, status: %s") % status)
c_id = self.db_info.compute_instance_id
nova_status = self.nova_client.servers.get(c_id).status
if nova_status in [InstanceStatus.ERROR, InstanceStatus.FAILED]:
raise TroveError(_("Server not active, status: %s") % nova_status)
return False
def validate(self, required_traits):
required_keys = set(required_traits)
provided_keys = set(self.payload.keys())
if not required_keys.issubset(provided_keys):
raise TroveError(_("The following required keys not defined for"
" notification %(name)s: %(keys)s")
% {'name': self.__class__.__name__,
'keys': list(required_keys - provided_keys)})
if 'server_type' not in self.payload:
raise TroveError(_("Notification %s must include a"
" 'server_type' for correct routing")
% self.__class__.__name__)
def _most_current_replica(self, old_master, replica_models):
last_txns = self._get_replica_txns(replica_models)
master_ids = [txn[1] for txn in last_txns if txn[1]]
if len(set(master_ids)) > 1:
raise TroveError(_("Replicas of %s not all replicating"
" from same master") % old_master.id)
return sorted(last_txns, key=lambda x: x[2], reverse=True)[0][0]
def _grow_cluster():
db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
cluster_head = next(
Instance.load(context, db_inst.id) for db_inst in db_instances
if db_inst.id not in new_instance_ids)
if not cluster_head:
raise TroveError(
_("Unable to determine existing Redis cluster"
" member"))
(cluster_head_ip,
cluster_head_port) = (self.get_guest(cluster_head).get_node_ip())
# Wait for cluster members to get to cluster-ready status.
if not self._all_instances_ready(new_instance_ids, cluster_id):
return
LOG.debug("All members ready, proceeding for cluster setup.")
new_insts = [
Instance.load(context, instance_id)
for instance_id in new_instance_ids
]
new_guests = map(self.get_guest, new_insts)
# Connect nodes to the cluster head
for guest in new_guests:
guest.cluster_meet(cluster_head_ip, cluster_head_port)
for guest in new_guests:
guest.cluster_complete()
def shrink(self, removal_ids):
LOG.debug("Shrinking cluster %s.", self.id)
self.validate_cluster_available()
cluster_info = self.db_info
cluster_info.update(task_status=ClusterTasks.SHRINKING_CLUSTER)
try:
removal_insts = [
inst_models.Instance.load(self.context, inst_id)
for inst_id in removal_ids
]
node_ids = [
Cluster.get_guest(instance).get_node_id_for_removal()
for instance in removal_insts
]
if None in node_ids:
raise TroveError(
_("Some nodes cannot be removed (check slots)"))
all_instances = (inst_models.DBInstance.find_all(
cluster_id=self.id, deleted=False).all())
remain_insts = [
inst_models.Instance.load(self.context, inst.id)
for inst in all_instances if inst.id not in removal_ids
]
map(lambda x: Cluster.get_guest(x).remove_nodes(node_ids),
remain_insts)
map(lambda x: x.update_db(cluster_id=None), removal_insts)
map(inst_models.Instance.delete, removal_insts)
return RedisCluster(self.context, cluster_info, self.ds,
self.ds_version)
finally:
cluster_info.update(task_status=ClusterTasks.NONE)
def ip_is_available(server):
LOG.info("Polling for ip addresses: $%s " % server.addresses)
if server.addresses != {}:
return True
elif (server.addresses == {}
and server.status != InstanceStatus.ERROR):
return False
elif (server.addresses == {}
and server.status == InstanceStatus.ERROR):
msg = _("Instance IP not available, instance (%s): "
"server had status (%s).")
LOG.error(msg % (self.id, server.status))
raise TroveError(status=server.status)
def ip_is_available(server):
LOG.info("Polling for ip addresses: $%s " % server.addresses)
if server.addresses != {}:
return True
elif (server.addresses == {}
and server.status != InstanceStatus.ERROR):
return False
elif (server.addresses == {}
and server.status == InstanceStatus.ERROR):
LOG.error(
_("Instance IP not available, "
"instance (%(instance)s): "
"server had status (%(status)s).") % {
'instance': self.id,
'status': server.status
})
raise TroveError(status=server.status)
def _create_server_volume_heat(self, flavor, image_id, datastore_manager,
volume_size, availability_zone):
LOG.debug(_("begin _create_server_volume_heat for id: %s") % self.id)
client = create_heat_client(self.context)
novaclient = create_nova_client(self.context)
cinderclient = create_cinder_client(self.context)
template_obj = template.load_heat_template(datastore_manager)
heat_template_unicode = template_obj.render()
try:
heat_template = heat_template_unicode.encode('ascii')
except UnicodeEncodeError:
LOG.error(_("heat template ascii encode issue"))
raise TroveError("heat template ascii encode issue")
parameters = {
"Flavor": flavor["name"],
"VolumeSize": volume_size,
"InstanceId": self.id,
"ImageId": image_id,
"DatastoreManager": datastore_manager,
"AvailabilityZone": availability_zone
}
stack_name = 'trove-%s' % self.id
client.stacks.create(stack_name=stack_name,
template=heat_template,
parameters=parameters)
stack = client.stacks.get(stack_name)
utils.poll_until(lambda: client.stacks.get(stack_name),
lambda stack: stack.stack_status in
['CREATE_COMPLETE', 'CREATE_FAILED'],
sleep_time=2,
time_out=HEAT_TIME_OUT)
resource = client.resources.get(stack.id, 'BaseInstance')
server = novaclient.servers.get(resource.physical_resource_id)
resource = client.resources.get(stack.id, 'DataVolume')
volume = cinderclient.volumes.get(resource.physical_resource_id)
volume_info = self._build_volume(volume)
self.update_db(compute_instance_id=server.id, volume_id=volume.id)
LOG.debug(_("end _create_server_volume_heat for id: %s") % self.id)
return server, volume_info
def __init__(self, context, **kwargs):
self.context = context
self.needs_end_notification = True
self.payload = {}
if 'request' in kwargs:
request = kwargs.pop('request')
self.payload.update({
'request_id': context.request_id,
'server_type': 'api',
'client_ip': request.remote_addr,
'server_ip': request.host,
'tenant_id': context.tenant,
})
elif 'request_id' not in kwargs:
raise TroveError(_("Notification %s must include 'request'"
" property") % self.__class__.__name__)
self.payload.update(kwargs)
def delete_backup(cls, context, backup_id):
#delete backup from swift
backup = trove.backup.models.Backup.get_by_id(context, backup_id)
try:
filename = backup.filename
if filename:
BackupTasks.delete_files_from_swift(context, filename)
except ValueError:
backup.delete()
except ClientException as e:
if e.http_status == 404:
# Backup already deleted in swift
backup.delete()
else:
LOG.exception("Exception deleting from swift. Details: %s" % e)
backup.state = trove.backup.models.BackupState.DELETE_FAILED
backup.save()
raise TroveError("Failed to delete swift objects")
else:
backup.delete()
def _compute_mem_allocations_mb(self, ramsize_quota_mb, enabled_services):
"""Couchbase 4.x and higher split the available memory quota between
data and index services.
If the indexing service is turned on the quota value must be at least
256MB.
Compute the index quota as 25% of the total and use the rest for data
services. Return '0' quota if the service is not enabled.
"""
if 'index' in enabled_services:
index_quota_mb = max(int(self.INDEX_MEM_RATIO * ramsize_quota_mb),
Couchbase4App.MIN_RAMSIZE_QUOTA_MB)
else:
index_quota_mb = 0
data_quota_mb = ramsize_quota_mb - index_quota_mb
if data_quota_mb < Couchbase4App.MIN_RAMSIZE_QUOTA_MB:
required = Couchbase4App.MIN_RAMSIZE_QUOTA_MB - data_quota_mb
raise TroveError(
_("Not enough memory for Couchbase services. "
"Additional %dMB is required.") % required)
return data_quota_mb, index_quota_mb
def _assert_nova_action_was_successful(self):
# Do check to make sure the status and flavor id are correct.
if str(self.instance.server.flavor['id']) != str(self.new_flavor_id):
msg = "Assertion failed! flavor_id=%s and not %s" \
% (self.instance.server.flavor['id'], self.new_flavor_id)
raise TroveError(msg)
def _assert_nova_status_is_ok(self):
# Make sure Nova thinks things went well.
if self.instance.server.status != "VERIFY_RESIZE":
msg = "Migration failed! status=%s and not %s" \
% (self.instance.server.status, 'VERIFY_RESIZE')
raise TroveError(msg)
def _log_and_raise(self, exc, message, task_status):
LOG.error(message)
LOG.error(exc)
LOG.error(traceback.format_exc())
self.update_db(task_status=task_status)
raise TroveError(message=message)
def _create_cluster():
# Fetch instances by cluster_id against instances table.
db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
instance_ids = [db_instance.id for db_instance in db_instances]
LOG.debug("Waiting for instances to get to cluster-ready status.")
# Wait for cluster members to get to cluster-ready status.
if not self._all_instances_ready(instance_ids, cluster_id):
raise TroveError(
_("Instances in cluster did not report "
"ACTIVE"))
LOG.debug("All members ready, proceeding for cluster setup.")
instances = [
Instance.load(context, instance_id)
for instance_id in instance_ids
]
cluster_ips = [self.get_ip(instance) for instance in instances]
instance_guests = [
self.get_guest(instance) for instance in instances
]
# Create replication user and password for synchronizing the
# galera cluster
replication_user = {
"name": self.CLUSTER_REPLICATION_USER,
"password": utils.generate_random_password(),
}
# Galera cluster name must be unique and be shorter than a full
# uuid string so we remove the hyphens and chop it off. It was
# recommended to be 16 chars or less.
# (this is not currently documented on Galera docs)
cluster_name = utils.generate_uuid().replace("-", "")[:16]
LOG.debug("Configuring cluster configuration.")
try:
# Set the admin password for all the instances because the
# password in the my.cnf will be wrong after the joiner
# instances syncs with the donor instance.
admin_password = str(utils.generate_random_password())
for guest in instance_guests:
guest.reset_admin_password(admin_password)
bootstrap = True
for instance in instances:
guest = self.get_guest(instance)
# render the conf.d/cluster.cnf configuration
cluster_configuration = self._render_cluster_config(
context, instance, ",".join(cluster_ips), cluster_name,
replication_user)
# push the cluster config and bootstrap the first instance
guest.install_cluster(replication_user,
cluster_configuration, bootstrap)
bootstrap = False
LOG.debug("Finalizing cluster configuration.")
for guest in instance_guests:
guest.cluster_complete()
except Exception:
LOG.exception(_("Error creating cluster."))
self.update_statuses_on_failure(cluster_id)
def _grow_cluster():
db_instances = DBInstance.find_all(cluster_id=cluster_id,
deleted=False).all()
existing_instances = [
Instance.load(context, db_inst.id) for db_inst in db_instances
if db_inst.id not in new_instance_ids
]
if not existing_instances:
raise TroveError(
_("Unable to determine existing cluster "
"member(s)"))
# get list of ips of existing cluster members
existing_cluster_ips = [
self.get_ip(instance) for instance in existing_instances
]
existing_instance_guests = [
self.get_guest(instance) for instance in existing_instances
]
# get the cluster context to setup new members
cluster_context = existing_instance_guests[0].get_cluster_context()
# Wait for cluster members to get to cluster-ready status.
if not self._all_instances_ready(new_instance_ids, cluster_id):
raise TroveError(
_("Instances in cluster did not report "
"ACTIVE"))
LOG.debug("All members ready, proceeding for cluster setup.")
# Get the new instances to join the cluster
new_instances = [
Instance.load(context, instance_id)
for instance_id in new_instance_ids
]
new_cluster_ips = [
self.get_ip(instance) for instance in new_instances
]
for instance in new_instances:
guest = self.get_guest(instance)
guest.reset_admin_password(cluster_context['admin_password'])
# render the conf.d/cluster.cnf configuration
cluster_configuration = self._render_cluster_config(
context, instance, ",".join(existing_cluster_ips),
cluster_context['cluster_name'],
cluster_context['replication_user'])
# push the cluster config and bootstrap the first instance
bootstrap = False
guest.install_cluster(cluster_context['replication_user'],
cluster_configuration, bootstrap)
self._check_cluster_for_root(context, existing_instances,
new_instances)
# apply the new config to all instances
for instance in existing_instances + new_instances:
guest = self.get_guest(instance)
# render the conf.d/cluster.cnf configuration
cluster_configuration = self._render_cluster_config(
context, instance,
",".join(existing_cluster_ips + new_cluster_ips),
cluster_context['cluster_name'],
cluster_context['replication_user'])
guest.write_cluster_configuration_overrides(
cluster_configuration)
for instance in new_instances:
guest = self.get_guest(instance)
guest.cluster_complete()
def test_invalid_error_message_format(self):
error = TroveError("test%999999sdb")
self.assertEqual(error.message, "test999999sdb")
def test_valid_error_message_format(self):
error = TroveError("%02d" % 1)
self.assertEqual(error.message, "01")
