def tag_shared_snapshots(snapshot_data, snap_ids): if self.accounts_with_create_permissions not in ["", None] and self.tag_shared_snapshots: for account in self.accounts_with_create_permissions: session_for_tagging = self.get_action_session(account=account, param_name=PARAM_SHARED_ACCOUNT_TAGGING_ROLENAME, logger=self._logger_) if session_for_tagging is None: self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account) continue try: ec2_client = get_client_with_retries(service_name="ec2", methods=[ "create_tags", "delete_tags" ], context=self._context_, region=self._region_, session=session_for_tagging, logger=self._logger_) for snap_id in snap_ids: tags = snapshot_data.get(snap_id, {}).get("tags", None) if tags is not None: self._logger_.info(INFO_SET_SNAPSHOT_TAGS_SHARED, safe_json(tags, indent=3), snap_id, account, self._region_) tagging.set_ec2_tags(ec2_client=ec2_client, resource_ids=[snap_id], tags=tags, logger=self._logger_) except Exception as ex: raise Exception(ERR_SETTING_SHARED_TAGS.format(account, str(ex)))
def _tag_instances(self): if len(self.over_utilized_instances) and self.get( PARAM_CPU_HIGH_TAGS) is not None: high_tags = self.build_tags_from_template( parameter_name=PARAM_CPU_HIGH_TAGS) self._logger_.info(INF_TAG_HIGH, ",".join(self.over_utilized_instances), high_tags) tagging.set_ec2_tags(ec2_client=self.ec2_client, resource_ids=self.over_utilized_instances, tags=high_tags, logger=self._logger_) if len(self.under_utilized_instances) and self.get( PARAM_CPU_LOW_TAGS) is not None: low_tags = self.build_tags_from_template( parameter_name=PARAM_CPU_LOW_TAGS) self._logger_.info(INF_TAG_LOW, ",".join(self.under_utilized_instances), low_tags) tagging.set_ec2_tags(ec2_client=self.ec2_client, resource_ids=self.under_utilized_instances, tags=low_tags, logger=self._logger_)
def delete_up_down_filter_tags(): tags = tags_to_delete() if len(tags) > 0: tagging.set_ec2_tags(ec2_client=self.ec2_client, tags=tags, can_delete=True, logger=self._logger_, resource_ids=[self.instance_id])
def set_snapshot_tags(snap, vol, dev): try: tags = get_tags_for_volume_snapshot(vol, dev) if self.set_snapshot_name: snapshot_name = self.build_str_from_template( parameter_name=PARAM_NAME, tag_variables={ TAG_PLACEHOLDER_INSTANCE_ID: self.instance_id, TAG_PLACEHOLDER_VOLUME_ID: volume }) if snapshot_name == "": dt = self._datetime_.utcnow() snapshot_name = SNAPSHOT_NAME.format( volume, dt.year, dt.month, dt.day, dt.hour, dt.minute) prefix = self.build_str_from_template( parameter_name=PARAM_SNAPSHOT_NAME_PREFIX, tag_variables={ TAG_PLACEHOLDER_INSTANCE_ID: self.instance_id, TAG_PLACEHOLDER_VOLUME_ID: volume }) snapshot_name = prefix + snapshot_name tags["Name"] = snapshot_name self._logger_.info(INFO_SNAPSHOT_NAME, snapshot_name) if len(tags) > 0: self._logger_.info(INFO_CREATE_TAGS, safe_json(tags, indent=3)) tagging.set_ec2_tags(ec2_client=self.ec2_client, resource_ids=[snap], tags=tags, can_delete=False, logger=self._logger_) if snap not in self.result["snapshots"]: self.result["snapshots"][snap] = {} self.result["snapshots"][snap]["tags"] = tags self._logger_.info(INFO_TAGS_CREATED) except Exception as ex: if self._dryrun_: self._logger_.debug(str(ex)) self.result["volumes"][volume]["create_tags"] = str(ex) else: raise ex
def set_source_snapshot_tags(copy_id): snapshot_tags = source_tags(copy_id, PARAM_SOURCE_TAGS) if len(snapshot_tags) == 0: return self._logger_.info(INF_CREATE_SOURCE_TAGS, snapshot_tags, self._account_) if len(snapshot_tags) > 0: tagging.set_ec2_tags(ec2_client=self.ec2_source_client, resource_ids=[self.source_snapshot_id], tags=snapshot_tags, logger=self._logger_) self._logger_.info(INF_TAGS_CREATED)
def set_volume_tags(volume_id, snap_id): tags = self.build_tags_from_template(parameter_name=PARAM_VOLUME_TAGS, tag_variables={ TAG_PLACEHOLDER_VOLUME_SNAPSHOT: snap_id }) if len(tags) > 0: try: tagging.set_ec2_tags(ec2_client=self.ec2_client, resource_ids=[volume_id], tags=tags, logger=self._logger_) self._logger_.info(INFO_SET_VOLUME_TAGS, safe_json(tags, indent=3), volume_id) except Exception as ex: raise Exception(ERR_SETTING_VOLUME_TAGS.format(self.instance_id, ex))
def tag_shared_source_snapshot(copy_id): # created tags for snapshots for shared snapshots in the source account of the shares snapshots snapshot_tags = source_tags(copy_id, PARAM_SOURCE_SHARED_BY_TAGS) if len(snapshot_tags ) == 0 or not self.tag_snapshots_in_source_account: return # only for snapshots that have been shared by other account if self.owner == self.get_account_for_task(): self._logger_.debug( "Account {} is owner, no tags set for snapshot {} in account of owner", self._account_, self.source_snapshot_id) return session_for_tagging = self.get_action_session( account=self.owner, param_name=PARAM_SOURCE_ACCOUNT_TAG_ROLE_NAME, logger=self._logger_) if session_for_tagging is None: self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, self.owner) return try: self._logger_.info(INF_CREATE_SHARED_ACCOUNT_SNAPSHOT_TAGS, snapshot_tags, self.source_snapshot_id, self.owner) ec2_client = get_client_with_retries( service_name="ec2", methods=["create_tags", "delete_tags"], context=self._context_, region=self.source_region, session=session_for_tagging, logger=self._logger_) tagging.set_ec2_tags(ec2_client=ec2_client, resource_ids=[self.source_snapshot_id], tags=snapshot_tags, logger=self._logger_) except Exception as ex: raise_exception(ERR_SETTING_SOURCE_SHARED_TAGS, self.owner, str(ex))
def tag_shared_snapshots(tags, snap_id): # creates tags for snapshots that have been shared in account the snapshots are shared with if len(tags) == 0 or not self.tag_snapshots_in_shared_accounts: return if self.accounts_with_create_permissions in ["", None]: return for account in self.accounts_with_create_permissions: session_for_tagging = self.get_action_session( account=account, param_name=PARAM_DESTINATION_ACCOUNT_TAG_ROLENAME, logger=self._logger_) if session_for_tagging is None: self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account) continue try: ec2_client = get_client_with_retries( service_name="ec2", methods=["create_tags", "delete_tags"], context=self._context_, region=self.get(PARAM_DESTINATION_REGION), session=session_for_tagging, logger=self._logger_) tagging.set_ec2_tags(ec2_client=ec2_client, resource_ids=[snap_id], tags=tags, logger=self._logger_) self._logger_.info(INF_CREATE_SHARED_TAGS, tags, account) except Exception as ex: raise_exception(ERR_SETTING_SHARED_TAGS, account, str(ex))
def execute(self): def get_tags_for_copied_snapshot(): snapshot_tags = ( self.copied_volume_tagfiter.pairs_matching_any_filter( self.snapshot.get("Tags", {}))) snapshot_tags[actions.marker_snapshot_tag_source_source_volume_id( )] = self.source_volume_id snapshot_tags.update( self.build_tags_from_template( parameter_name=PARAM_SNAPSHOT_TAGS, region=self.source_region, tag_variables={ TAG_PLACEHOLDER_SOURCE_SNAPSHOT_ID: self.source_snapshot_id, TAG_PLACEHOLDER_SOURCE_REGION: self.source_region, TAG_PLACEHOLDER_OWNER_ACCOUNT: self.owner, TAG_PLACEHOLDER_SOURCE_VOLUME: self.source_volume_id })) snapshot_tags[Ec2CopySnapshotAction.marker_tag_source_snapshot_id( )] = self.source_snapshot_id snapshot_tags[actions.marker_snapshot_tag_source_source_volume_id( )] = self.source_volume_id return snapshot_tags def get_source_snapshot(): ec2 = services.create_service( "ec2", session=self._session_, service_retry_strategy=get_default_retry_strategy( "ec2", context=self._context_)) snapshot = ec2.get(services.ec2_service.SNAPSHOTS, region=self.source_region, RestorableByUserIds=["self"], Filters=[{ "Name": "snapshot-id", "Values": [self.source_snapshot_id] }]) return snapshot def should_copy_snapshot(): snapshot = get_source_snapshot() # source snapshot was already deleted by tasks that were in wait for execution list if snapshot is None: self.result["not-longer-available"] = True self._logger_.info(INF_WARNING_NO_SNAPSHOT, self.source_snapshot_id) return False # get tags from the snapshot, these must have contain the mark_as_copied tag and this tag must contain the same # copy serial number as the snapshot that was in the selected resource for this task instance source_snapshot_tags = snapshot.get( "Tags", {}) if snapshot is not None else {} marked_as_copied_tag = Ec2CopySnapshotAction.marker_tag_copied_to( self._task_) if marked_as_copied_tag in source_snapshot_tags: snapshot_copy_data = json.loads( source_snapshot_tags[marked_as_copied_tag]) else: snapshot_copy_data = {} if snapshot_copy_data.get( COPY_SERIAL_NUMBER, "") != self.snapshot.get(COPY_SERIAL_NUMBER): self._logger_.info(INF_COPIED_BY_OTHER, snapshot_copy_data.get(TAG_REGION, ""), snapshot_copy_data(COPY_SERIAL_NUMBER, "")) self.result["already-copied"] = True self.result["copied-data"] = snapshot_copy_data return False return True # logged information self._logger_.info("{}, version {}", self.properties[ACTION_TITLE], self.properties[ACTION_VERSION]) self._logger_.info(INF_ACCOUNT_SNAPSHOT, self.source_snapshot_id, self._account_, self.source_region, self._destination_region_) self._logger_.debug("Snapshot : {}", self.snapshot) boto_call = "copy_snapshot" try: # setup argument for CopySnapshot call args = { "SourceRegion": self.source_region, "SourceSnapshotId": self.source_snapshot_id } if not should_copy_snapshot(): return self.result if self.encrypted: args["Encrypted"] = True self.result["encrypted"] = True if self.kms_key_id not in ["", None]: args["KmsKeyId"] = self.kms_key_id if self._dryrun_: args["DryRun"] = True source_description = self.snapshot.get("Description", "") description_variables = { TAG_PLACEHOLDER_SOURCE_SNAPSHOT_ID: self.source_snapshot_id, TAG_PLACEHOLDER_SOURCE_REGION: self.source_region, TAG_PLACEHOLDER_OWNER_ACCOUNT: self.owner, TAG_PLACEHOLDER_SOURCE_VOLUME: self.source_volume_id, TAG_PLACEHOLDER_SOURCE_DESCRIPTION: source_description } args["Description"] = self.build_str_from_template( parameter_name=PARAM_SNAPSHOT_DESCRIPTION, region=self.source_region, tag_variables=description_variables) if args["Description"] == "": args["Description"] = source_description # start the copy resp = self.ec2_destination_client.copy_snapshot_with_retries( **args) # id of the copy copy_snapshot_id = resp.get("SnapshotId") self._logger_.info(INF_SNAPSHOT_COPIED, self.source_snapshot_id, self._destination_region_, copy_snapshot_id) self.result[boto_call] = resp self.result["copy-snapshot-id"] = copy_snapshot_id # update the tag that marks the snapshot as being copied boto_call = "create_tags (source)" copied_tag_name = Ec2CopySnapshotAction.marker_tag_copied_to( self._task_) copy_data_tag = { copied_tag_name: safe_json({ TAG_REGION: self._destination_region_, COPY_SERIAL_NUMBER: self.snapshot.get(COPY_SERIAL_NUMBER, ""), TAG_COPIED_BY_TASK: self.get(ACTION_PARAM_TASK_ID, ""), TAG_COPY_SNAPSHOT_ID: copy_snapshot_id }) } self.ec2_source_client.create_tags_with_retries( Resources=[self.source_snapshot_id], Tags=tag_key_value_list(copy_data_tag)) # set tags on the copy boto_call = "create_tags (target)" tags = get_tags_for_copied_snapshot() self._logger_.info(INF_CREATE_COPIED_TAGS, tags, self._account_) if len(tags) > 0: tagging.set_ec2_tags(ec2_client=self.ec2_destination_client, resource_ids=[copy_snapshot_id], tags=tags, logger=self._logger_) self.result["tags"] = tags self._logger_.info(INF_TAGS_CREATED) except Exception as ex: if self._dryrun_: self._logger_.debug(str(ex)) self.result[boto_call] = str(ex) return self.result else: raise ex self.result[METRICS_DATA] = build_action_metrics(self, CopiedSnapshots=1) return self.result
def set_ec2_instance_tags_with_event_loop_check(self, instance_ids, tags_to_set, client=None, region=None): def get_instances(): ec2 = services.create_service("ec2", session=self._session_, service_retry_strategy=get_default_retry_strategy("ec2", context=self._context_)) return list(ec2.describe(services.ec2_service.INSTANCES, InstanceIds=instance_ids, region=region if region is not None else self._region_, tags=True, select="Reservations[*].Instances[].{Tags:Tags,InstanceId:InstanceId}")) def get_ec2_client(): if client is not None: return client methods = ["create_tags", "delete_tags"] return get_client_with_retries("ec2", methods=methods, region=region, session=self._session_, logger=self._logger_) try: if len(tags_to_set) > 0: tagged_instances = instance_ids[:] # before setting the tags check if these tags won't trigger a new execution of the task causing a loop task_events = self.get(ACTION_PARAM_EVENTS, {}) task_change_events = task_events.get(handlers.ec2_tag_event_handler.EC2_TAG_EVENT_SOURCE, {}).get( handlers.TAG_CHANGE_EVENT, []) if handlers.ec2_tag_event_handler.EC2_CHANGED_INSTANCE_TAGS_EVENT in task_change_events: tag_name = os.getenv(handlers.ENV_AUTOMATOR_TAG_NAME) tag_filter_str = self.get(ACTION_PARAM_TAG_FILTER, None) tag_filter = TagFilterExpression(tag_filter_str) if tag_filter_str not in ["", None, "None"] else None for instance in get_instances(): # tags currently on instance instance_tags = instance.get("Tags", {}) # tags that have updated values when setting the tags deleted_tags = {t: tags_to_set[t] for t in tags_to_set if tags_to_set[t] == tagging.TAG_DELETE and t in instance_tags} new_tags = {t: tags_to_set[t] for t in tags_to_set if t not in instance_tags and tags_to_set[t] != tagging.TAG_DELETE} updated_tags = {t: tags_to_set[t] for t in tags_to_set if tags_to_set[t] != tagging.TAG_DELETE and t in instance_tags and instance_tags[t] != tags_to_set[t]} updated_tags.update(new_tags) # if there are updates if any([len(t) > 0 for t in [new_tags, updated_tags, deleted_tags]]): # this will be the new set of tags for the instance updated_instance_tags = copy.deepcopy(instance_tags) for t in deleted_tags: del updated_instance_tags[t] for t in updated_tags: updated_instance_tags[t] = updated_tags[t] # test if we have a tag filter and if the filter matches the new tags if tag_filter is not None: updated_tags_used_in_filter = set(updated_tags).intersection(tag_filter.get_filter_keys()) # tags updated that are in the tag filter if len(updated_tags_used_in_filter) > 0: # test if updated tags trigger the task if tag_filter.is_match(updated_instance_tags): self._logger_.warning(WARN_LOOP_TAG_TAGFILTER, tags_to_set, tag_filter_str, instance["InstanceId"]) tagged_instances.remove(instance["InstanceId"]) # if no tag filter then check if the tag with the Ops Automator tasks does contain the name of the task else: task_list = updated_instance_tags.get(tag_name, "") if tag_name in updated_tags and self._task_ in tagging.split_task_list(task_list): self._logger_.warning(WARN_LOOP_TAG, tags_to_set, task_list, tag_name, instance["InstanceId"]) tagged_instances.remove(instance["InstanceId"]) if len(tagged_instances) > 0: tagging.set_ec2_tags(ec2_client=get_ec2_client(), resource_ids=tagged_instances, tags=tags_to_set) except Exception as ex: self._logger_.error(ERR_SET_TAGS, ','.join(instance_ids), str(ex))