def create_heat_snapshot(context): filters = {'deleted': False} instances = Clients(context).nova().servers.list(search_opts=filters) snapshot = [] for instance in instances: LOG.debug(_("Creating heat snapshot, iterating instance %s ."), instance.id) instance_id = instance.id metadata = Clients(context).nova().servers.get(instance_id).metadata if 'dr_state' in metadata and metadata['dr_state'] == "ready": snap = instance_snapshot(context, instance) snapshot.append(snap) return snapshot
def __init__(self, context, is_unit_test=False): super(HeatFlameTemplate, self).__init__(context) self.dr_image_snapshot_ids = [] self.dr_protected_server_names = [] self.dr_volumes = [] # each has : volume name only ! self.dr_replicated_volumes = [] self.dr_networks = [] self.credentials = {} self.dr_keynames = [] self.context = context self.clients = Clients(context) LOG.debug("heat_flame_template . initializing ") if is_unit_test is True: password = '******' else: importutils.import_module('keystoneclient.middleware.auth_token') password = cfg.CONF.keystone_authtoken.admin_password self.credentials = { 'user': self.context.username, 'password': password, 'project': self.context.tenant, 'auth_url': self.context.auth_url, 'auth_token': self.context.auth_token } LOG.debug("heat_flame_template credentials: user: %s, password : %s," "project: %s, auth_url %s , auth_token %s" % (self.credentials['user'], self.credentials['password'], self.credentials['project'], self.credentials['auth_url'], self.credentials['auth_token']))
def remote_copy_ssh_key(context, key): LOG.debug(" key name = %s" % (key)) ssh_key_name = key.id # id is actually key name . see : novaclient/v1_1/keypairs.py/def id(self) LOG.debug("key name by dot = %s" % (ssh_key_name)) key_data = key.public_key LOG.debug("key data = %s." % (key_data)) if ssh_key_name is not None: try: dr_remote_nova_client = Clients(context).remote_nova() LOG.debug('.... connected to remote Nova....') if (is_remote_existing_keyname(dr_remote_nova_client, ssh_key_name) is False): LOG.debug('....copying ssh key name : "%s"' % (ssh_key_name)) dr_remote_nova_client.keypairs.create(ssh_key_name, key_data) LOG.debug('.... public key copied to remote Nova') except Exception: ssh_key_name = None LOG.error('write public key data to remote site/swift failed..') return ssh_key_name
def __init__(self, context): self.clients = Clients(context) self.cntx = context self._instances = [] self._volumes = [] self._replicated_volumes = [] self._keypairs = []
def __init__(self, context): self.clients = Clients(context) self._snap_id = None self._image_id = None self._name = None self._resource_id = None self.data_block_size_bytes = CONF.backup_snapshot_object_size
def _imagecopy(self, context, instance, container_name, action_excution): backup_rec = {} action_excution.set_status(context, 'uploaded to swift') swift_conn = Clients(context).swift() headers = {'X-Container-Meta-dr_state': 'processing'} image = self.clients.glance().images.get(self._image_id) # take the checksum as unique id global_container_image_id = image._info['checksum'] image_response = image.data() image_response_data = StringIO.StringIO() for chunk in image_response: image_response_data.write(chunk) image_response_data.seek(0, os.SEEK_SET) chunks = 0 while True: data = image_response_data.read(self.data_block_size_bytes) data_offset = image_response_data.tell() LOG.debug("uploading image offset %s chunks %s" % (data_offset, chunks)) if data == '': break try: swift_conn.put_object(container_name, global_container_image_id + "_" + str(chunks), data, content_length=len(data)) chunks += 1 except socket.error as err: dr_state = 'DR image backup failed' action_excution.set_status(context, dr_state) raise exception.SwiftConnectionFailed(reason=str(err)) dr_state = 'Protected' backup_rec["metadata"] = instance.metadata backup_rec["image_id"] = global_container_image_id backup_rec["instance_name"] = self._name backup_rec["meta"] = image.to_dict() backup_rec["chunks"] = chunks action_excution.set_status(context, dr_state) return dr_state, backup_rec
def instance_volumes_list(context, instance): LOG.debug("_instance_volumes_list %s %s." % (instance, instance.id)) bdms = Clients(context).nova().volumes.get_server_volumes(instance.id) if not bdms: LOG.debug(_("Instance %s is not attached."), instance.id) mountpoints = [] for bdm in bdms: LOG.debug("bdm %s %s." % (bdm, dir(bdm))) volume_id = bdm.volumeId assigned_mountpoint = bdm.device if volume_id is not None: mountpoints.append( translate_attachment_detail_view(volume_id, instance.id, assigned_mountpoint)) return mountpoints
def failover(context, container_name): swift_conn = Clients(context).swift() LOG.debug("failover container name %s" % (container_name)) # TODO(Oshrit): load dynamically template_generator = heat_flame_template.HeatFlameTemplate(context) # Loop over metadata file, load actions swift_meta, actions_metadata = swift_conn.get_object( container_name, "metadata") actions = jsonutils.loads(actions_metadata) policy_status = False try: for recover_action in actions["actions"]: action_resource = recover_action["resource"] db_action = recover_action["action"] action_extra_data = recover_action["execution_data"] # Instantiate action class action_obj = action.load_action_driver(db_action["class_name"], context=context) if action_obj.is_using_global_container(): action_container_name =\ get_global_container_name(context.tenant, "") else: action_container_name = container_name # Invoke recover method on action class passing resource id policy_status = action_obj.failover(context, action_resource["id"], action_extra_data, action_container_name) # Invoke failover for each resource-action action_obj.generate_template(context, template_generator) except Exception, e: policy_status = False LOG.debug(e) LOG.error("resource %s could not be recovered using action %s." "Verify that the resource is a valid resource" % (db_action["id"], db_action["name"]))
def instance_snapshot_reverse_volume(context, instance): LOG.debug("In snapshot of instance %s ." % instance) try: image = Clients(context).glance().images.get(instance.image['id']) flavor = Clients(context).nova().flavors.get(instance.flavor['id']) volumes = instance_volumes_list(context, instance) networks = Clients(context).remote_neutron().\ list_networks()['networks'] # =================================================================== LOG.debug("In snapshot, remote networks %s %s." % (networks, dir(networks))) for net in networks: LOG.debug("In snapshot, network %s %s." % (net, dir(net))) if net['name'] == 'private': remote_private_net_id = net['id'] remote_private_net_subnet_id = net['subnets'][0] if net['name'] == 'public': remote_public_net_id = net['id'] LOG.debug("In remote private network id = %s." % (remote_private_net_id)) volume_mapping = [] for v in volumes: LOG.debug("In snapshot, instance's volume %s %s." % (v, dir(v))) c_client = Clients(context).cinder() vol = c_client.volumes.get(v['volumeId']) if ('dr_state' in vol.metadata and vol.metadata['dr_state'] == "ready"): # check if the consistency group is present if vol.metadata['consist_grp_name']: volume_mapping.append({ 'id': vol.metadata['dr_backup_id'], 'device': v['device'], 'consist_grp': vol.metadata['consist_grp_name'] }) else: volume_mapping.append({ 'id': vol.metadata['dr_backup_id'], 'device': v['device'] }) # =================================================================== # network_mapping = [] # for net in networks: # network = self.clients.nova().networks.get(net['id']) # LOG.debug("In snapshot, instance's networks details %s." % network) # =================================================================== return { 'inst': instance, 'img': image, 'flavor': flavor, 'volumes': volume_mapping, 'networks': instance.addresses, 'remote_private_net_id': remote_private_net_id, 'remote_private_net_subnet_id': remote_private_net_subnet_id, 'remote_public_net_id': remote_public_net_id } except Exception, e: LOG.error("Failed getting master images %s" % (e)) LOG.error(traceback.format_exc()) LOG.error(sys.exc_info()[0])
def generate_instance_and_net_section(context, snap, out): instance = snap['inst'] image = snap['img'] flavor = snap['flavor'] ssh_key_name = snap['key_name'] instance_name = instance.name image_name = image.name flavor_id = flavor.id fl = Clients(context).nova().flavors.get(flavor_id) flavor_name = fl.name instance_has_floating = False LOG.debug("instance networks %s" % snap['networks']) for net in snap['networks'].values(): for address in net: LOG.debug("instance networks loop %s %s %s." % (address, type(address), dir(address))) if address['OS-EXT-IPS:type'] == 'floating': instance_has_floating = True port_name = "%s_port" % instance_name if "resources" not in out or out["resources"] is None: out["resources"] = {} if ssh_key_name is not None: inst_properties = { "image": image_name, "flavor": flavor_name, "key_name": ssh_key_name, "networks": [{ "port": { "get_resource": port_name } }], } else: inst_properties = { "image": image_name, "flavor": flavor_name, "networks": [{ "port": { "get_resource": port_name } }], } out["resources"][instance_name] = { "type": "OS::Nova::Server", "properties": inst_properties } fixed_ips = [] fixed_ips.append({'subnet_id': snap['remote_private_net_subnet_id']}) out["resources"][port_name] = { "type": "OS::Neutron::Port", "properties": { "network_id": snap['remote_private_net_id'], "fixed_ips": fixed_ips, } } if instance_has_floating: floating_ip_name = "%s_floating_ip" % instance_name out["resources"][floating_ip_name] = { "type": "OS::Neutron::FloatingIP", "properties": { "floating_network_id": snap['remote_public_net_id'], "port_id": { "get_resource": port_name }, } } LOG.debug("instance and net info = %s" % (out))
def instance_snapshot(context, instance): LOG.debug("In snapshot of instance %s ." % instance) try: image = Clients(context).glance().images.get(instance.image['id']) flavor = Clients(context).nova().flavors.get(instance.flavor['id']) key = Clients(context).nova().keypairs.get(instance.key_name) # returned object is keypair class LOG.debug("key_name type = %s" % type(key)) ssh_key_name = remote_copy_ssh_key(context, key) LOG.debug("ssh key name (before write to snap) = %s" % (ssh_key_name)) volumes = instance_volumes_list(context, instance) networks = Clients(context).remote_neutron().\ list_networks()['networks'] # =================================================================== LOG.debug("In snapshot, remote networks %s %s." % (networks, dir(networks))) for net in networks: LOG.debug("In snapshot, network %s %s." % (net, dir(net))) if net['name'] == 'private': remote_private_net_id = net['id'] remote_private_net_subnet_id = net['subnets'][0] if net['name'] == 'public': remote_public_net_id = net['id'] LOG.debug("In remote private network id = %s." % (remote_private_net_id)) volume_mapping = [] for v in volumes: LOG.debug("In snapshot, instance's volume %s %s." % (v, dir(v))) c_client = Clients(context).cinder() # cinder_client.cinderclient(context) vol = c_client.volumes.get(v['volumeId']) if ('dr_state' in vol.metadata and vol.metadata['dr_state'] == "ready"): volume_mapping.append({ 'id': v['volumeId'], 'device': v['device'], 'dr_backup_id': vol.metadata['dr_backup_id'] }) # =================================================================== # network_mapping = [] # for net in networks: # network = self.clients.nova().networks.get(net['id']) # LOG.debug("In snapshot, instance's networks details %s ." # % network) # =================================================================== return { 'inst': instance, 'img': image, 'flavor': flavor, 'key_name': ssh_key_name, 'volumes': volume_mapping, 'networks': instance.addresses, 'remote_private_net_id': remote_private_net_id, 'remote_private_net_subnet_id': remote_private_net_subnet_id, 'remote_public_net_id': remote_public_net_id } except Exception, e: LOG.error("Failed getting master images %s" % (e)) LOG.error(traceback.format_exc()) LOG.error(sys.exc_info()[0]) return None
def __init__(self, context): self.clients = Clients(context) self._name = None self._id = None self._backup_id = None self._resource_id = None
def protect(self, cnxt): # Define container_name, Swift create if container does not exists container_name = get_policy_execution_container_name( cnxt.tenant, self.name) swift_conn = Clients(cnxt).swift() # container = swift.swift_create_container(request, name) headers = {'X-Container-Meta-dr_state': 'processing'} swift_conn.put_container(container_name, headers) LOG.debug("put_container %s " % (container_name)) # Create a global container for the tenant in case it does not exists if action.Action.is_using_global_container: swift_conn.put_container( get_global_container_name(cnxt.tenant, self.name), {'X-Container-Meta-dr_state': 'ready'}) # create workload_policy_execution instance protect_execution = { 'workload_policy_id': self.id, 'status': 'Creating', } workload_action_excution =\ api.workload_policy_excution_create(cnxt, protect_execution) # TODO(load dynamic) template_generator = heat_flame_template.HeatFlameTemplate(cnxt) try: if self.consistent_protect: self.pre_protect(cnxt, workload_action_excution) # iterate on actions protect_statuses = [] metadata = {} metadata["workload_policy_name"] = self.name policy_status = True LOG.debug("workload_policy protect, policy_status = %s" % (policy_status)) metadata["actions"] = [] for db_action in self.actions: try: # instantiate action class action_obj = action.load(db_action, context=cnxt) if action_obj.is_using_global_container(): action_container_name =\ get_global_container_name(cnxt.tenant, self.name) else: action_container_name = container_name # invoke protect method on action class passing resource id status, protect_data =\ action_obj.protect(cnxt, workload_action_excution.id, db_action.resource.id, action_container_name) LOG.debug("workload_policy protect, status = %s" % (status)) policy_status = policy_status and (status == "Protected") LOG.debug("workload_policy protect, policy_status = %s" % (policy_status)) action_obj.generate_template(cnxt, template_generator) # save result of invocation in relation # to workload_policy_execution protect_statuses.\ append({"action id": db_action.action.id, "action_name": db_action.action.name, "resource_id": db_action.resource.id, "resource_name": db_action.resource.name, "status": status}) db_action.execution_data = protect_data metadata["actions"].append(db_action) LOG.debug("action metadata %s " % (metadata)) except Exception, e: policy_status = False exc = sys.exc_info() LOG.error(traceback.format_exception(*exc)) LOG.error("resource %s could not be protected using action" "%s. Verify that the resource is a valid " " resource" % (db_action.resource.id, db_action.action.name)) except Exception, e: policy_status = False LOG.debug(e) LOG.error("Workload could not be protected")
action_container_name) # Invoke failover for each resource-action action_obj.generate_template(context, template_generator) except Exception, e: policy_status = False LOG.debug(e) LOG.error("resource %s could not be recovered using action %s." "Verify that the resource is a valid resource" % (db_action["id"], db_action["name"])) if not policy_status: return policy_status keypairs = Clients(context).nova().keypairs.list() for keypair in keypairs: template_generator.add_keypair(KeyPairResource(keypair.name)) # TODO(Oshrit): no need to run HEAT if one of the # resources failed to restore swift_meta, template = swift_conn.get_object(container_name, "template.yaml") LOG.debug("template.yaml = %s " % template) adjusted_template =\ template_generator.process_recover_template(template) LOG.debug("adjusted template = %s " % adjusted_template) stack_name = container_name
def _snapshot(self, context, instance, container_name, action_excution): # metadata = instance.metadata n_client = self.clients.nova() snapshot_name = instance.name + "_snapshot" snapshot_metadata = instance.metadata instance_snapshot = instance.create_image(snapshot_name, instance.metadata) self._snap_id = instance_snapshot action_excution.set_status(context, 'taking snapshot') local_snapshot = n_client.images.get(instance_snapshot) LOG.debug("checking instance snapshot %s %s " % (local_snapshot.status, local_snapshot.progress)) while (local_snapshot.status == "SAVING"): greenthread.sleep(1) local_snapshot = n_client.images.get(instance_snapshot) backup_rec = {} if local_snapshot.status == "ACTIVE": action_excution.set_status(context, 'uploading to swift') swift_conn = Clients(context).swift() headers = {'X-Container-Meta-dr_state': 'processing'} image = self.clients.glance().images.get(instance_snapshot) image_response = image.data() image_response_data = StringIO.StringIO() for chunk in image_response: image_response_data.write(chunk) image_response_data.seek(0, os.SEEK_SET) chunks = 0 while True: data = image_response_data.read(self.data_block_size_bytes) data_offset = image_response_data.tell() LOG.debug("uploading offset %s chunks %s" % (data_offset, chunks)) if data == '': break try: swift_conn.put_object(container_name, instance_snapshot + "_" + str(chunks), data, content_length=len(data)) chunks += 1 except socket.error as err: raise exception.SwiftConnectionFailed(reason=str(err)) dr_state = 'Protected' backup_rec["metadata"] = instance.metadata backup_rec["snap_id"] = self._snap_id backup_rec["instance_name"] = self._name backup_rec["meta"] = image.to_dict() backup_rec["chunks"] = chunks self._cleanup(context, n_client, self._snap_id) else: dr_state = 'DR clone backup failed' action_excution.set_status(context, dr_state) return dr_state, backup_rec