def tag_shared_snapshots(snapshot_data, snap_ids):
if self.accounts_with_create_permissions not in ["", None] and self.tag_shared_snapshots:
for account in self.accounts_with_create_permissions:
session_for_tagging = self.get_action_session(account=account,
param_name=PARAM_SHARED_ACCOUNT_TAGGING_ROLENAME,
logger=self._logger_)
if session_for_tagging is None:
self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account)
continue
try:
ec2_client = get_client_with_retries(service_name="ec2",
methods=[
"create_tags",
"delete_tags"
],
context=self._context_,
region=self._region_,
session=session_for_tagging,
logger=self._logger_)
for snap_id in snap_ids:
tags = snapshot_data.get(snap_id, {}).get("tags", None)
if tags is not None:
self._logger_.info(INFO_SET_SNAPSHOT_TAGS_SHARED, safe_json(tags, indent=3), snap_id, account,
self._region_)
tagging.set_ec2_tags(ec2_client=ec2_client,
resource_ids=[snap_id],
tags=tags,
logger=self._logger_)
except Exception as ex:
raise Exception(ERR_SETTING_SHARED_TAGS.format(account, str(ex)))
def _tag_instances(self):
if len(self.over_utilized_instances) and self.get(
PARAM_CPU_HIGH_TAGS) is not None:
high_tags = self.build_tags_from_template(
parameter_name=PARAM_CPU_HIGH_TAGS)
self._logger_.info(INF_TAG_HIGH,
",".join(self.over_utilized_instances),
high_tags)
tagging.set_ec2_tags(ec2_client=self.ec2_client,
resource_ids=self.over_utilized_instances,
tags=high_tags,
logger=self._logger_)
if len(self.under_utilized_instances) and self.get(
PARAM_CPU_LOW_TAGS) is not None:
low_tags = self.build_tags_from_template(
parameter_name=PARAM_CPU_LOW_TAGS)
self._logger_.info(INF_TAG_LOW,
",".join(self.under_utilized_instances),
low_tags)
tagging.set_ec2_tags(ec2_client=self.ec2_client,
resource_ids=self.under_utilized_instances,
tags=low_tags,
logger=self._logger_)
def delete_up_down_filter_tags():
tags = tags_to_delete()
if len(tags) > 0:
tagging.set_ec2_tags(ec2_client=self.ec2_client,
tags=tags,
can_delete=True,
logger=self._logger_,
resource_ids=[self.instance_id])
def set_snapshot_tags(snap, vol, dev):
try:
tags = get_tags_for_volume_snapshot(vol, dev)
if self.set_snapshot_name:
snapshot_name = self.build_str_from_template(
parameter_name=PARAM_NAME,
tag_variables={
TAG_PLACEHOLDER_INSTANCE_ID: self.instance_id,
TAG_PLACEHOLDER_VOLUME_ID: volume
})
if snapshot_name == "":
dt = self._datetime_.utcnow()
snapshot_name = SNAPSHOT_NAME.format(
volume, dt.year, dt.month, dt.day, dt.hour,
dt.minute)
prefix = self.build_str_from_template(
parameter_name=PARAM_SNAPSHOT_NAME_PREFIX,
tag_variables={
TAG_PLACEHOLDER_INSTANCE_ID: self.instance_id,
TAG_PLACEHOLDER_VOLUME_ID: volume
})
snapshot_name = prefix + snapshot_name
tags["Name"] = snapshot_name
self._logger_.info(INFO_SNAPSHOT_NAME, snapshot_name)
if len(tags) > 0:
self._logger_.info(INFO_CREATE_TAGS,
safe_json(tags, indent=3))
tagging.set_ec2_tags(ec2_client=self.ec2_client,
resource_ids=[snap],
tags=tags,
can_delete=False,
logger=self._logger_)
if snap not in self.result["snapshots"]:
self.result["snapshots"][snap] = {}
self.result["snapshots"][snap]["tags"] = tags
self._logger_.info(INFO_TAGS_CREATED)
except Exception as ex:
if self._dryrun_:
self._logger_.debug(str(ex))
self.result["volumes"][volume]["create_tags"] = str(ex)
else:
raise ex
def set_source_snapshot_tags(copy_id):
snapshot_tags = source_tags(copy_id, PARAM_SOURCE_TAGS)
if len(snapshot_tags) == 0:
return
self._logger_.info(INF_CREATE_SOURCE_TAGS, snapshot_tags,
self._account_)
if len(snapshot_tags) > 0:
tagging.set_ec2_tags(ec2_client=self.ec2_source_client,
resource_ids=[self.source_snapshot_id],
tags=snapshot_tags,
logger=self._logger_)
self._logger_.info(INF_TAGS_CREATED)
def set_volume_tags(volume_id, snap_id):
tags = self.build_tags_from_template(parameter_name=PARAM_VOLUME_TAGS,
tag_variables={
TAG_PLACEHOLDER_VOLUME_SNAPSHOT: snap_id
})
if len(tags) > 0:
try:
tagging.set_ec2_tags(ec2_client=self.ec2_client,
resource_ids=[volume_id],
tags=tags,
logger=self._logger_)
self._logger_.info(INFO_SET_VOLUME_TAGS, safe_json(tags, indent=3), volume_id)
except Exception as ex:
raise Exception(ERR_SETTING_VOLUME_TAGS.format(self.instance_id, ex))
def tag_shared_source_snapshot(copy_id):
# created tags for snapshots for shared snapshots in the source account of the shares snapshots
snapshot_tags = source_tags(copy_id, PARAM_SOURCE_SHARED_BY_TAGS)
if len(snapshot_tags
) == 0 or not self.tag_snapshots_in_source_account:
return
# only for snapshots that have been shared by other account
if self.owner == self.get_account_for_task():
self._logger_.debug(
"Account {} is owner, no tags set for snapshot {} in account of owner",
self._account_, self.source_snapshot_id)
return
session_for_tagging = self.get_action_session(
account=self.owner,
param_name=PARAM_SOURCE_ACCOUNT_TAG_ROLE_NAME,
logger=self._logger_)
if session_for_tagging is None:
self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, self.owner)
return
try:
self._logger_.info(INF_CREATE_SHARED_ACCOUNT_SNAPSHOT_TAGS,
snapshot_tags, self.source_snapshot_id,
self.owner)
ec2_client = get_client_with_retries(
service_name="ec2",
methods=["create_tags", "delete_tags"],
context=self._context_,
region=self.source_region,
session=session_for_tagging,
logger=self._logger_)
tagging.set_ec2_tags(ec2_client=ec2_client,
resource_ids=[self.source_snapshot_id],
tags=snapshot_tags,
logger=self._logger_)
except Exception as ex:
raise_exception(ERR_SETTING_SOURCE_SHARED_TAGS, self.owner,
str(ex))
def tag_shared_snapshots(tags, snap_id):
# creates tags for snapshots that have been shared in account the snapshots are shared with
if len(tags) == 0 or not self.tag_snapshots_in_shared_accounts:
return
if self.accounts_with_create_permissions in ["", None]:
return
for account in self.accounts_with_create_permissions:
session_for_tagging = self.get_action_session(
account=account,
param_name=PARAM_DESTINATION_ACCOUNT_TAG_ROLENAME,
logger=self._logger_)
if session_for_tagging is None:
self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account)
continue
try:
ec2_client = get_client_with_retries(
service_name="ec2",
methods=["create_tags", "delete_tags"],
context=self._context_,
region=self.get(PARAM_DESTINATION_REGION),
session=session_for_tagging,
logger=self._logger_)
tagging.set_ec2_tags(ec2_client=ec2_client,
resource_ids=[snap_id],
tags=tags,
logger=self._logger_)
self._logger_.info(INF_CREATE_SHARED_TAGS, tags, account)
except Exception as ex:
raise_exception(ERR_SETTING_SHARED_TAGS, account, str(ex))
def execute(self):
def get_tags_for_copied_snapshot():
snapshot_tags = (
self.copied_volume_tagfiter.pairs_matching_any_filter(
self.snapshot.get("Tags", {})))
snapshot_tags[actions.marker_snapshot_tag_source_source_volume_id(
)] = self.source_volume_id
snapshot_tags.update(
self.build_tags_from_template(
parameter_name=PARAM_SNAPSHOT_TAGS,
region=self.source_region,
tag_variables={
TAG_PLACEHOLDER_SOURCE_SNAPSHOT_ID:
self.source_snapshot_id,
TAG_PLACEHOLDER_SOURCE_REGION: self.source_region,
TAG_PLACEHOLDER_OWNER_ACCOUNT: self.owner,
TAG_PLACEHOLDER_SOURCE_VOLUME: self.source_volume_id
}))
snapshot_tags[Ec2CopySnapshotAction.marker_tag_source_snapshot_id(
)] = self.source_snapshot_id
snapshot_tags[actions.marker_snapshot_tag_source_source_volume_id(
)] = self.source_volume_id
return snapshot_tags
def get_source_snapshot():
ec2 = services.create_service(
"ec2",
session=self._session_,
service_retry_strategy=get_default_retry_strategy(
"ec2", context=self._context_))
snapshot = ec2.get(services.ec2_service.SNAPSHOTS,
region=self.source_region,
RestorableByUserIds=["self"],
Filters=[{
"Name": "snapshot-id",
"Values": [self.source_snapshot_id]
}])
return snapshot
def should_copy_snapshot():
snapshot = get_source_snapshot()
# source snapshot was already deleted by tasks that were in wait for execution list
if snapshot is None:
self.result["not-longer-available"] = True
self._logger_.info(INF_WARNING_NO_SNAPSHOT,
self.source_snapshot_id)
return False
# get tags from the snapshot, these must have contain the mark_as_copied tag and this tag must contain the same
# copy serial number as the snapshot that was in the selected resource for this task instance
source_snapshot_tags = snapshot.get(
"Tags", {}) if snapshot is not None else {}
marked_as_copied_tag = Ec2CopySnapshotAction.marker_tag_copied_to(
self._task_)
if marked_as_copied_tag in source_snapshot_tags:
snapshot_copy_data = json.loads(
source_snapshot_tags[marked_as_copied_tag])
else:
snapshot_copy_data = {}
if snapshot_copy_data.get(
COPY_SERIAL_NUMBER,
"") != self.snapshot.get(COPY_SERIAL_NUMBER):
self._logger_.info(INF_COPIED_BY_OTHER,
snapshot_copy_data.get(TAG_REGION, ""),
snapshot_copy_data(COPY_SERIAL_NUMBER, ""))
self.result["already-copied"] = True
self.result["copied-data"] = snapshot_copy_data
return False
return True
# logged information
self._logger_.info("{}, version {}", self.properties[ACTION_TITLE],
self.properties[ACTION_VERSION])
self._logger_.info(INF_ACCOUNT_SNAPSHOT, self.source_snapshot_id,
self._account_, self.source_region,
self._destination_region_)
self._logger_.debug("Snapshot : {}", self.snapshot)
boto_call = "copy_snapshot"
try:
# setup argument for CopySnapshot call
args = {
"SourceRegion": self.source_region,
"SourceSnapshotId": self.source_snapshot_id
}
if not should_copy_snapshot():
return self.result
if self.encrypted:
args["Encrypted"] = True
self.result["encrypted"] = True
if self.kms_key_id not in ["", None]:
args["KmsKeyId"] = self.kms_key_id
if self._dryrun_:
args["DryRun"] = True
source_description = self.snapshot.get("Description", "")
description_variables = {
TAG_PLACEHOLDER_SOURCE_SNAPSHOT_ID: self.source_snapshot_id,
TAG_PLACEHOLDER_SOURCE_REGION: self.source_region,
TAG_PLACEHOLDER_OWNER_ACCOUNT: self.owner,
TAG_PLACEHOLDER_SOURCE_VOLUME: self.source_volume_id,
TAG_PLACEHOLDER_SOURCE_DESCRIPTION: source_description
}
args["Description"] = self.build_str_from_template(
parameter_name=PARAM_SNAPSHOT_DESCRIPTION,
region=self.source_region,
tag_variables=description_variables)
if args["Description"] == "":
args["Description"] = source_description
# start the copy
resp = self.ec2_destination_client.copy_snapshot_with_retries(
**args)
# id of the copy
copy_snapshot_id = resp.get("SnapshotId")
self._logger_.info(INF_SNAPSHOT_COPIED, self.source_snapshot_id,
self._destination_region_, copy_snapshot_id)
self.result[boto_call] = resp
self.result["copy-snapshot-id"] = copy_snapshot_id
# update the tag that marks the snapshot as being copied
boto_call = "create_tags (source)"
copied_tag_name = Ec2CopySnapshotAction.marker_tag_copied_to(
self._task_)
copy_data_tag = {
copied_tag_name:
safe_json({
TAG_REGION:
self._destination_region_,
COPY_SERIAL_NUMBER:
self.snapshot.get(COPY_SERIAL_NUMBER, ""),
TAG_COPIED_BY_TASK:
self.get(ACTION_PARAM_TASK_ID, ""),
TAG_COPY_SNAPSHOT_ID:
copy_snapshot_id
})
}
self.ec2_source_client.create_tags_with_retries(
Resources=[self.source_snapshot_id],
Tags=tag_key_value_list(copy_data_tag))
# set tags on the copy
boto_call = "create_tags (target)"
tags = get_tags_for_copied_snapshot()
self._logger_.info(INF_CREATE_COPIED_TAGS, tags, self._account_)
if len(tags) > 0:
tagging.set_ec2_tags(ec2_client=self.ec2_destination_client,
resource_ids=[copy_snapshot_id],
tags=tags,
logger=self._logger_)
self.result["tags"] = tags
self._logger_.info(INF_TAGS_CREATED)
except Exception as ex:
if self._dryrun_:
self._logger_.debug(str(ex))
self.result[boto_call] = str(ex)
return self.result
else:
raise ex
self.result[METRICS_DATA] = build_action_metrics(self,
CopiedSnapshots=1)
return self.result
def set_ec2_instance_tags_with_event_loop_check(self, instance_ids, tags_to_set, client=None, region=None):
def get_instances():
ec2 = services.create_service("ec2", session=self._session_,
service_retry_strategy=get_default_retry_strategy("ec2", context=self._context_))
return list(ec2.describe(services.ec2_service.INSTANCES,
InstanceIds=instance_ids,
region=region if region is not None else self._region_,
tags=True,
select="Reservations[*].Instances[].{Tags:Tags,InstanceId:InstanceId}"))
def get_ec2_client():
if client is not None:
return client
methods = ["create_tags",
"delete_tags"]
return get_client_with_retries("ec2",
methods=methods,
region=region,
session=self._session_,
logger=self._logger_)
try:
if len(tags_to_set) > 0:
tagged_instances = instance_ids[:]
# before setting the tags check if these tags won't trigger a new execution of the task causing a loop
task_events = self.get(ACTION_PARAM_EVENTS, {})
task_change_events = task_events.get(handlers.ec2_tag_event_handler.EC2_TAG_EVENT_SOURCE, {}).get(
handlers.TAG_CHANGE_EVENT, [])
if handlers.ec2_tag_event_handler.EC2_CHANGED_INSTANCE_TAGS_EVENT in task_change_events:
tag_name = os.getenv(handlers.ENV_AUTOMATOR_TAG_NAME)
tag_filter_str = self.get(ACTION_PARAM_TAG_FILTER, None)
tag_filter = TagFilterExpression(tag_filter_str) if tag_filter_str not in ["", None, "None"] else None
for instance in get_instances():
# tags currently on instance
instance_tags = instance.get("Tags", {})
# tags that have updated values when setting the tags
deleted_tags = {t: tags_to_set[t] for t in tags_to_set if
tags_to_set[t] == tagging.TAG_DELETE and t in instance_tags}
new_tags = {t: tags_to_set[t] for t in tags_to_set if
t not in instance_tags and tags_to_set[t] != tagging.TAG_DELETE}
updated_tags = {t: tags_to_set[t] for t in tags_to_set if
tags_to_set[t] != tagging.TAG_DELETE and t in instance_tags and instance_tags[t] !=
tags_to_set[t]}
updated_tags.update(new_tags)
# if there are updates
if any([len(t) > 0 for t in [new_tags, updated_tags, deleted_tags]]):
# this will be the new set of tags for the instance
updated_instance_tags = copy.deepcopy(instance_tags)
for t in deleted_tags:
del updated_instance_tags[t]
for t in updated_tags:
updated_instance_tags[t] = updated_tags[t]
# test if we have a tag filter and if the filter matches the new tags
if tag_filter is not None:
updated_tags_used_in_filter = set(updated_tags).intersection(tag_filter.get_filter_keys())
# tags updated that are in the tag filter
if len(updated_tags_used_in_filter) > 0:
# test if updated tags trigger the task
if tag_filter.is_match(updated_instance_tags):
self._logger_.warning(WARN_LOOP_TAG_TAGFILTER,
tags_to_set,
tag_filter_str,
instance["InstanceId"])
tagged_instances.remove(instance["InstanceId"])
# if no tag filter then check if the tag with the Ops Automator tasks does contain the name of the task
else:
task_list = updated_instance_tags.get(tag_name, "")
if tag_name in updated_tags and self._task_ in tagging.split_task_list(task_list):
self._logger_.warning(WARN_LOOP_TAG, tags_to_set, task_list, tag_name, instance["InstanceId"])
tagged_instances.remove(instance["InstanceId"])
if len(tagged_instances) > 0:
tagging.set_ec2_tags(ec2_client=get_ec2_client(),
resource_ids=tagged_instances,
tags=tags_to_set)
except Exception as ex:
self._logger_.error(ERR_SET_TAGS, ','.join(instance_ids), str(ex))
