def validate_tagfilter(tag_filter, action_name): """ Tests if tags are supported by the resources for the action. If this is nit the case then the use of tag filters is not possible and an exception is raised :param tag_filter: Tag filter value :param action_name: Name of the action :return: Filter if tags are supported and the filter can be used, otherwise an exception is raised """ if tag_filter is None or tag_filter == "": return None action_properties = actions.get_action_properties(action_name) resource_supports_tags = action_properties.get( actions.ACTION_RESOURCES) in services.create_service( action_properties[actions.ACTION_SERVICE]).resources_with_tags # resource does not allow tags, so tagfilters can not be used if not resource_supports_tags: raise ValueError( MSG_NO_TAG_FILTER.format( action_properties[actions.ACTION_RESOURCES], tag_filter, action_name)) # destructive actions can deny use of wildcards for tag name if not action_properties.get(actions.ACTION_ALLOW_TAGFILTER_WILDCARD, True): if "".join([s.strip() for s in tag_filter.split("=")[0:1] ]) in ["*", "**", "*="]: raise ValueError( MSG_NO_WILDCARS_TAG_FILTER_ALLOWED.format( tag_filter, action_name)) return tag_filter
def get_instances(): ec2 = services.create_service("ec2", session=self._session_, service_retry_strategy=get_default_retry_strategy("ec2", context=self._context_)) return list(ec2.describe(services.ec2_service.INSTANCES, InstanceIds=instance_ids, region=region if region is not None else self._region_, tags=True, select="Reservations[*].Instances[].{Tags:Tags,InstanceId:InstanceId}"))
def create_service(self, name, service_type): """ Create a service. @param name: Service name @param service_type: Service type @return: An ApiService object """ return services.create_service(self._get_resource_root(), name, service_type, self.name)
def setup_environment(stack_name): try: cfn = create_service("cloudformation", session=boto3.Session()) print_verbose( "Retrieving Lambda resource \"SchedulerDefault\" from stack \"{}\"", stack_name) stack = cfn.get(services.cloudformation_service.STACKS, StackName=stack_name) lambda_name = [ o["OutputValue"] for o in stack["Outputs"] if o["OutputKey"] == "OpsAutomatorLambdaFunctionStandard" ][0] print_verbose("Lambda physical resource id is {}", lambda_name) lambda_service = create_service("lambda", session=boto3.Session()) lambda_function = lambda_service.get("Function", FunctionName=lambda_name) environment = lambda_function["Configuration"]["Environment"][ "Variables"] print_verbose("Lambda environment variables are:") for env_var in environment: os.environ[env_var] = environment[env_var] print_verbose("{}=\"{}\"", env_var, environment[env_var]) role_resource = cfn.get("StackResource", StackName=stack_name, LogicalResourceId="OpsAutomatorLambdaRole") role = boto3.client("iam").get_role( RoleName=role_resource["PhysicalResourceId"]).get("Role", {}) os.environ["ROLE_ARN"] = role["Arn"] except Exception as ex: print("error setting up environment {}".format(str(ex)))
def __init__(self, stack_name, region=None, owned=True): self._stack_id = None self._stack_outputs = None self._stack_resources = None self.region = region if region is not None else boto3.Session( ).region_name self.cfn_client = boto3.Session().client("cloudformation", region_name=self.region) self.cnf_service = services.create_service("cloudformation") self.owned = owned self.stack_name = stack_name
def service_regions(cls, service_name): """ Returns available regions for a service :param service_name: Name of the service :return: list of regions in which the service is available """ available_regions = _service_regions.get(service_name) if available_regions is not None: return available_regions available_regions = services.create_service( service_name).service_regions() _service_regions[service_name] = available_regions return available_regions
def _account_service_sessions(self, service_name): """ Returns a list of service instances for each handled account/role :return: """ account = self._event.get(handlers.HANDLER_SELECT_ARGUMENTS, {}).get(handlers.HANDLER_EVENT_ACCOUNT) retry_strategy = get_default_retry_strategy(service=service_name, context=self._context) if account is not None: if account == AwsService.get_aws_account(): yield services.create_service(service_name=service_name) else: for role in self.task[actions.ACTION_CROSS_ACCOUNT]: if AwsService.account_from_role_arn(role) == account: yield services.create_service( service_name=service_name, role_arn=role, service_retry_strategy=retry_strategy) else: self._logger.error(MSG_NO_CROSS_ACCOUNT_ROLE, self.task[handlers.TASK_NAME], account) else: if self.task.get(handlers.TASK_THIS_ACCOUNT, True): yield services.create_service( service_name=service_name, service_retry_strategy=retry_strategy) for role in self.task.get(handlers.TASK_CROSS_ACCOUNT_ROLES, []): yield services.create_service( service_name=service_name, role_arn=role, service_retry_strategy=retry_strategy)
def get_source_snapshot(): ec2 = services.create_service( "ec2", session=self._session_, service_retry_strategy=get_default_retry_strategy( "ec2", context=self._context_)) snapshot = ec2.get(services.ec2_service.SNAPSHOTS, region=self.source_region, RestorableByUserIds=["self"], Filters=[{ "Name": "snapshot-id", "Values": [self.source_snapshot_id] }]) return snapshot
def _parse_services(self, template_json): for s in template_json['services']: name = s['name'] service = services.create_service(name) self.services.append(service) for c in s['components']: component = Component(c['name'], c['type'], c['cardinality']) service.add_component(component) if 'users' in s: for u in s['users']: user = User(u['name'], u['password'], u['groups']) service.add_user(user) configs = self._parse_configurations(s) for config in configs: service.add_configuration(config)
def setup_tag_filter_parameter(): # test if the resource support tags action_resources = self.action_properties.get(actions.ACTION_RESOURCES) service = self.action_properties[actions.ACTION_SERVICE] self._resource_supports_tags = action_resources and action_resources in services.create_service( service).resources_with_tags if not self._resource_supports_tags: return # task tag filter self._template_parameters[configuration.CONFIG_TAG_FILTER] = { "Type": "String", "Description": PARAM_DESCRIPTION_TAG_FILTER.format(self.scheduler_tag_name) } if not (self.action_properties.get(actions.ACTION_ALLOW_TAGFILTER_WILDCARD, True)): self._template_parameters[configuration.CONFIG_TAG_FILTER]["Description"] += PARAM_DESCRIPTION_TAG_FILER_NO_WILDCARD self._parameter_labels[configuration.CONFIG_TAG_FILTER] = {"default": PARAM_LABEL_FILTER} self._parameter_groups[0]["Parameters"].insert(2, configuration.CONFIG_TAG_FILTER)
def execute(self): def volume_has_active_snapshots(ec2_service, vol_id): # test if the snapshot with the ids that were returned from the CreateSnapshot API call exists and are completed volume_snapshots = list( ec2_service.describe(services.ec2_service.SNAPSHOTS, OwnerIds=["self"], region=self.instance["Region"], Filters=[ { "Name": "volume-id", "Values": [vol_id] } ])) active = [s["SnapshotId"] for s in volume_snapshots if s.get("State", "") == "pending"] if len(active) > 0: self._logger_.info(INFO_PENDING_SNAPSHOTS, vol_id, ",".join(active)) return True return False self._logger_.info("{}, version {}", self.properties[ACTION_TITLE], self.properties[ACTION_VERSION]) self._logger_.info(INFO_START_SNAPSHOT_ACTION, self.instance_id, self._account_, self._region_, self._task_) self._logger_.debug("Instance block device mappings are {}", self.instance["BlockDeviceMappings"]) ec2 = services.create_service("ec2", session=self._session_, service_retry_strategy=get_default_retry_strategy("ec2", context=self._context_)) if self.volume_tag_filter is not None: volume_data = ec2.describe(services.ec2_service.VOLUMES, VolumeIds=list(self.volumes.keys()), tags=True, region=self._region_) volume_tags = {k["VolumeId"]: k.get("Tags", {}) for k in list(volume_data)} else: volume_tags = {} if self.backup_root_device: if self.root_volume is None: self._logger_.warning(WARN_ROOT_NOT_FOUND, self.instance_id, ",".join(self.volumes)) else: if self.volume_tag_filter is None or self.volume_tag_filter.is_match(volume_tags.get(self.root_volume, {})): if volume_has_active_snapshots(ec2, self.root_volume): self._logger_.error(ERR_SNAPSHOT_PENDING, self.root_volume) else: self.create_volume_snapshot(self.root_volume) else: self._logger_.info(INF_SKIP_VOLUME_TAG_FILTER, self.root_volume, volume_tags.get(self.root_volume, {})) if self.backup_data_devices: for volume in [v for v in self.volumes if v != self.root_volume]: if self.volume_tag_filter is None or self.volume_tag_filter.is_match(volume_tags.get(volume, {})): if volume_has_active_snapshots(ec2, volume): self._logger_.error(ERR_SNAPSHOT_PENDING, volume) else: self.create_volume_snapshot(volume) else: self._logger_.info(INF_SKIP_VOLUME_TAG_FILTER, volume, volume_tags.get(volume, {})) self.result["start-time"] = self._datetime_.now().isoformat() self.result[METRICS_DATA] = build_action_metrics( action=self, CreatedSnapshots=len(list(self.result.get("volumes", {}).values())), SnapshotsSizeTotal=sum( [volume.get("create_snapshot", {}).get("VolumeSize") for volume in list(self.result.get("volumes", {}).values())])) return self.result
def is_completed(self, snapshot_create_data): def delete_source_after_copy(): self._logger_.info(INF_DELETING_SNAPSHOT, self.source_snapshot_id) self.ec2_source_client.delete_snapshot_with_retries( SnapshotId=self.source_snapshot_id) self._logger_.info(INF_SNAPSHOT_DELETED, self.source_snapshot_id, self.source_region) def source_tags(copy_id, source_tags_param): snapshot_tags = {} snapshot_tags.update( self.build_tags_from_template( parameter_name=source_tags_param, region=self.source_region, tag_variables={ TAG_PLACEHOLDER_COPIED_SNAPSHOT_ID: copy_id, TAG_PLACEHOLDER_COPIED_REGION: self._destination_region_ })) return snapshot_tags def set_source_snapshot_tags(copy_id): snapshot_tags = source_tags(copy_id, PARAM_SOURCE_TAGS) if len(snapshot_tags) == 0: return self._logger_.info(INF_CREATE_SOURCE_TAGS, snapshot_tags, self._account_) if len(snapshot_tags) > 0: tagging.set_ec2_tags(ec2_client=self.ec2_source_client, resource_ids=[self.source_snapshot_id], tags=snapshot_tags, logger=self._logger_) self._logger_.info(INF_TAGS_CREATED) def grant_create_volume_permissions(snap_id): if self.accounts_with_create_permissions is not None and len( self.accounts_with_create_permissions) > 0: args = { "CreateVolumePermission": { "Add": [{ "UserId": a.strip() } for a in self.accounts_with_create_permissions] }, "SnapshotId": snap_id } try: self.ec2_destination_client.modify_snapshot_attribute_with_retries( **args) self._logger_.info( INF_SETTING_CREATE_VOLUME_PERMISSIONS, ", ".join(self.accounts_with_create_permissions)) except Exception as ex: raise_exception(ERR_SETTING_CREATE_VOLUME_PERMISSIONS, self.accounts_with_create_permissions, ex) def tag_shared_snapshots(tags, snap_id): # creates tags for snapshots that have been shared in account the snapshots are shared with if len(tags) == 0 or not self.tag_snapshots_in_shared_accounts: return if self.accounts_with_create_permissions in ["", None]: return for account in self.accounts_with_create_permissions: session_for_tagging = self.get_action_session( account=account, param_name=PARAM_DESTINATION_ACCOUNT_TAG_ROLENAME, logger=self._logger_) if session_for_tagging is None: self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account) continue try: ec2_client = get_client_with_retries( service_name="ec2", methods=["create_tags", "delete_tags"], context=self._context_, region=self.get(PARAM_DESTINATION_REGION), session=session_for_tagging, logger=self._logger_) tagging.set_ec2_tags(ec2_client=ec2_client, resource_ids=[snap_id], tags=tags, logger=self._logger_) self._logger_.info(INF_CREATE_SHARED_TAGS, tags, account) except Exception as ex: raise_exception(ERR_SETTING_SHARED_TAGS, account, str(ex)) def tag_shared_source_snapshot(copy_id): # created tags for snapshots for shared snapshots in the source account of the shares snapshots snapshot_tags = source_tags(copy_id, PARAM_SOURCE_SHARED_BY_TAGS) if len(snapshot_tags ) == 0 or not self.tag_snapshots_in_source_account: return # only for snapshots that have been shared by other account if self.owner == self.get_account_for_task(): self._logger_.debug( "Account {} is owner, no tags set for snapshot {} in account of owner", self._account_, self.source_snapshot_id) return session_for_tagging = self.get_action_session( account=self.owner, param_name=PARAM_SOURCE_ACCOUNT_TAG_ROLE_NAME, logger=self._logger_) if session_for_tagging is None: self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, self.owner) return try: self._logger_.info(INF_CREATE_SHARED_ACCOUNT_SNAPSHOT_TAGS, snapshot_tags, self.source_snapshot_id, self.owner) ec2_client = get_client_with_retries( service_name="ec2", methods=["create_tags", "delete_tags"], context=self._context_, region=self.source_region, session=session_for_tagging, logger=self._logger_) tagging.set_ec2_tags(ec2_client=ec2_client, resource_ids=[self.source_snapshot_id], tags=snapshot_tags, logger=self._logger_) except Exception as ex: raise_exception(ERR_SETTING_SOURCE_SHARED_TAGS, self.owner, str(ex)) if snapshot_create_data.get("already-copied", False): self._logger_.info(INF_COMPLETE_ALREADY_COPIED, self.source_snapshot_id) return self.result if snapshot_create_data.get("not-longer-available", False): self._logger_.info(INF_COMPLETED_NOT_LONGER_AVAILABLE, self.source_snapshot_id) return self.result # create service instance to test if snapshot exists ec2 = services.create_service( "ec2", session=self._session_, service_retry_strategy=get_default_retry_strategy( "ec2", context=self._context_)) copy_snapshot_id = snapshot_create_data["copy-snapshot-id"] # test if the snapshot with the id that was returned from the CopySnapshot API call exists and is completed copied_snapshot = ec2.get(services.ec2_service.SNAPSHOTS, region=self._destination_region_, OwnerIds=["self"], Filters=[{ "Name": "snapshot-id", "Values": [copy_snapshot_id] }]) if copied_snapshot is not None: self._logger_.debug(INF_CHECK_COMPLETED_RESULT, copied_snapshot) state = copied_snapshot[ "State"] if copied_snapshot is not None else None if copied_snapshot is None or state == SNAPSHOT_STATE_PENDING: self._logger_.info(INF_COPY_PENDING, copy_snapshot_id, self._destination_region_) return None if state == SNAPSHOT_STATE_ERROR: copied_tag_name = Ec2CopySnapshotAction.marker_tag_copied_to( self._task_) self.ec2_source_client.delete_tags_with_retries( Resources=[self.source_snapshot_id], Tags=[{ "Key": copied_tag_name }]) raise_exception(ERR_COPY_SNAPSHOT) if state == SNAPSHOT_STATE_COMPLETED: self._logger_.info(INF_COPY_COMPLETED, self.source_snapshot_id, self.source_region, copy_snapshot_id, self._destination_region_) grant_create_volume_permissions(copy_snapshot_id) tag_shared_snapshots(snapshot_create_data.get("tags", {}), copy_snapshot_id) tag_shared_source_snapshot(copy_snapshot_id) if self.delete_after_copy: delete_source_after_copy() else: set_source_snapshot_tags(copy_snapshot_id) # wait there for 15 seconds as count the limit for max number of concurrent snapshot copies # by the EC2 service is sometimes delayed time.sleep(5) return copied_snapshot return None
def handle_request(self): """ Handles the select resources request. Creates new actions for resources found for a task :return: Results of handling the request """ def filter_by_action_filter(srv, used_role, r): filter_method = getattr(self.action_class, actions.SELECT_AND_PROCESS_RESOURCE_METHOD, None) if filter_method is not None: r = filter_method(srv, self._logger, self._resource_name, r, self._context, self.task, used_role) if r is None: self._logger.debug( DEBUG_FILTER_METHOD, self.action_class.__name__, actions.SELECT_AND_PROCESS_RESOURCE_METHOD) return None else: self._logger.debug( DEBUG_FILTERED_RESOURCE, self.action_class.__name__, actions.SELECT_AND_PROCESS_RESOURCE_METHOD, safe_json(r, indent=3)) return r def is_selected_resource(aws_service, resource, used_role, taskname, tags_filter, does_resource_supports_tags): # No tags then just use filter method if any if not does_resource_supports_tags: self._logger.debug(DEBUG_RESOURCE_NO_TAGS, resource) return filter_by_action_filter(srv=aws_service, used_role=used_role, r=resource) tags = resource.get("Tags", {}) # name of the tag that holds the list of tasks for this resource tagname = self._task_tag if tags_filter is None: # test if name of the task is in list of tasks in tag value if (tagname not in tags) or (taskname not in tagging.split_task_list( tags[tagname])): self._logger.debug( DEBUG_RESOURCE_NOT_SELECTED, safe_json(resource, indent=2), taskname, ','.join( ["'{}'='{}'".format(t, tags[t]) for t in tags])) return None self._logger.debug(DEBUG_SELECTED_BY_TASK_NAME_IN_TAG_VALUE, safe_json(resource, indent=2), tagname, taskname) else: # using a tag filter, * means any tag if tags_filter != tagging.tag_filter_set.WILDCARD_CHAR: # test if there are any tags matching the tag filter if not TagFilterExpression(tags_filter).is_match(tags): self._logger.debug( DEBUG_RESOURCE_NOT_SELECTED_TAG_FILTER, safe_json(resource, indent=2), taskname, ','.join([ "'{}'='{}'".format(t, tags[t]) for t in tags ])) return None self._logger.debug(DEBUG_SELECTED_BY_TAG_FILTER, safe_json(resource, indent=2), tags, tag_filter_str, taskname) else: self._logger.debug(DEBUG_SELECTED_WILDCARD_TAG_FILTER, safe_json(resource, indent=2), taskname) return filter_by_action_filter(srv=aws_service, used_role=used_role, r=resource) return filter_by_action_filter(srv=aws_service, used_role=used_role, r=resource) def resource_batches(resources): """ Returns resources as chunks of size items. If the class has an optional custom aggregation function then the resources are aggregated first using this function before applying the batch size :param resources: resources to process :return: Generator for blocks of resource items """ aggregate_func = getattr(self.action_class, actions.CUSTOM_AGGREGATE_METHOD, None) for i in aggregate_func( resources, self.task_parameters, self._logger) if aggregate_func is not None else [ resources ]: if self.batch_size is None: yield i else: first = 0 while first < len(i): yield i[first:first + self.batch_size] first += self.batch_size def setup_tag_filtering(t_name): # get optional tag filter no_select_by_tags = self.action_properties.get( actions.ACTION_NO_TAG_SELECT, False) if no_select_by_tags: tag_filter_string = tagging.tag_filter_set.WILDCARD_CHAR else: tag_filter_string = self.task.get(handlers.TASK_TAG_FILTER) # set if only a single task is required for selecting the resources, it is used to optimise the select select_tag = None if tag_filter_string is None: self._logger.debug(DEBUG_SELECT_BY_TASK_NAME, self._resource_name, self._task_tag, t_name) select_tag = self._task_tag elif tag_filter_string == tagging.tag_filter_set.WILDCARD_CHAR: self._logger.debug(DEBUG_SELECT_ALL_RESOURCES, self._resource_name) else: self._logger.debug(DEBUG_TAG_FILTER_USED_TO_SELECT_RESOURCES, self._resource_name) # build the tag expression that us used to filter the resources tag_filter_expression = TagFilterExpression(tag_filter_string) # the keys of the used tags tag_filter_expression_tag_keys = list( tag_filter_expression.get_filter_keys()) # if there is only a single tag then we can optimize by just filtering on that specific tag if len(tag_filter_expression_tag_keys) == 1 and \ tagging.tag_filter_set.WILDCARD_CHAR not in tag_filter_expression_tag_keys[0]: select_tag = tag_filter_expression_tag_keys[0] return select_tag, tag_filter_string def add_aggregated(aggregated_resources): # create tasks action for aggregated resources , optionally split in batch size chunks for ra in resource_batches(aggregated_resources): if self._check_can_execute(ra): action_item = self.actions_tracking.add_task_action( task=self.task, assumed_role=assumed_role, action_resources=ra, task_datetime=self.task_dt, source=self.source, task_group=self.task_group) self._logger.debug(DEBUG_ADDED_AGGREGATED_RESOURCES_TASK, action_item[handlers.TASK_TR_ID], len(ra), self._resource_name, self.task[handlers.TASK_NAME]) self._logger.debug("Added item\n{}", safe_json(action_item, indent=3)) yield action_item def add_as_individual(resources): for ri in resources: # task action for each selected resource if self._check_can_execute([ri]): action_item = self.actions_tracking.add_task_action( task=self.task, assumed_role=assumed_role, action_resources=ri, task_datetime=self.task_dt, source=self.source, task_group=self.task_group) self._logger.debug(DEBUG_ADD_SINGLE_RESOURCE_TASK, action_item[handlers.TASK_TR_ID], self._resource_name, self.task[handlers.TASK_NAME]) self._logger.debug("Added item\n{}", safe_json(action_item, indent=3)) yield action_item try: task_items = [] start = datetime.now() self._logger.debug(DEBUG_EVENT, safe_json(self._event, indent=3)) self._logger.debug(DEBUG_ACTION, safe_json(self.action_properties, indent=3)) self._logger.info(INFO_SELECTED_RESOURCES, self._resource_name, self.service, self.task[handlers.TASK_NAME]) self._logger.info(INFO_AGGR_LEVEL, self.aggregation_level) task_level_aggregated_resources = [] args = self._build_describe_argument() service_resource_with_tags = services.create_service( self.service).resources_with_tags if self._resource_name == "": supports_tags = len(service_resource_with_tags) != 0 else: supports_tags = self._resource_name.lower() in [ r.lower() for r in service_resource_with_tags ] args["tags"] = supports_tags self._logger.info(INFO_USE_TAGS_TO_SELECT, "R" if supports_tags else "No r") task_name = self.task[handlers.TASK_NAME] count_resource_items = 0 selected_resource_items = 0 select_on_tag, tag_filter_str = setup_tag_filtering(task_name) filter_func = getattr(self.action_class, actions.FILTER_RESOURCE_METHOD, None) # timer to guard selection time and log warning if getting close to lambda timeout if self._context is not None: self.start_timer(REMAINING_TIME_AFTER_DESCRIBE) try: for assumed_role in self._task_assumed_roles(): retry_strategy = get_default_retry_strategy( service=self.service, context=self._context) service = services.create_service( service_name=self.service, service_retry_strategy=retry_strategy, role_arn=assumed_role) if self.is_timed_out(): break # contains resources for account account_level_aggregated_resources = [] self._logger.info(INFO_ACCOUNT, service.aws_account) if assumed_role not in [None, ""]: self._logger.info(INFO_ASSUMED_ROLE, assumed_role) for region in self._regions: # test for timeouts if self.is_timed_out(): break # handle region passed in the event if region is not None: args["region"] = region else: if "region" in args: del args["region"] # resources can be passed in the invent by event handlers all_resources = self._event.get( handlers.HANDLER_SELECT_RESOURCES, None) if all_resources is None: # actions can have an optional method to select resources action_custom_describe_function = getattr( self.action_class, "describe_resources", None) if action_custom_describe_function is not None and self.use_custom_select: all_resources = action_custom_describe_function( service, self.task, region) else: # select resources from the service self._logger.debug(DEBUG_SELECT_PARAMETERS, self._resource_name, self.service, args) # selecting a list of all resources in this account/region all_resources = list( service.describe( self._resource_name, filter_func=filter_func, select_on_tag=select_on_tag, **args)) # test for timeout if self.is_timed_out(): break count_resource_items += len(all_resources) self._logger.info(INFO_RESOURCES_FOUND, len(all_resources)) # select resources that are processed by the task selected_resources = [] for sr in all_resources: sel = is_selected_resource( aws_service=service, resource=sr, used_role=assumed_role, taskname=task_name, tags_filter=tag_filter_str, does_resource_supports_tags=supports_tags) if sel is not None: selected_resources.append(sel) selected_resource_items += len(selected_resources) # display found and selected resources if len(all_resources) > 0: self._logger.info(INFO_RESOURCES_SELECTED, len(selected_resources)) if len(selected_resources) == 0: continue # delete tags if not needed by the action if not self.keep_tags: for res in selected_resources: if "Tags" in res: del res["Tags"] # add resources to total list of resources for this task if self.aggregation_level == actions.ACTION_AGGREGATION_TASK: task_level_aggregated_resources += selected_resources # add resources to list of resources for this account if self.aggregation_level == actions.ACTION_AGGREGATION_ACCOUNT: account_level_aggregated_resources += selected_resources # add batch(es) of resources for this region if self.aggregation_level == actions.ACTION_AGGREGATION_REGION and len( selected_resources) > 0: task_items += list( add_aggregated(selected_resources)) # no aggregation, add each individual resource if self.aggregation_level == actions.ACTION_AGGREGATION_RESOURCE and len( selected_resources) > 0: task_items += list( add_as_individual(selected_resources)) # at the end of the region loop, check if aggregated resources for account need to be added if self.aggregation_level == actions.ACTION_AGGREGATION_ACCOUNT and len( account_level_aggregated_resources) > 0: task_items += list( add_aggregated(account_level_aggregated_resources)) # at the end of the accounts loop, check if aggregated resources for task need to be added if self.aggregation_level == actions.ACTION_AGGREGATION_TASK and len( task_level_aggregated_resources) > 0: task_items += list( add_aggregated(task_level_aggregated_resources)) except Exception as ex: raise_exception(ERR_SELECTING_TASK_RESOURCES, self.task[handlers.TASK_NAME], ex) finally: if self._timer is not None: # cancel time used avoid timeouts when selecting resources self._timer.cancel() if self.is_timed_out(): raise_exception(ERR_TIMEOUT_SELECTING_RESOURCES, self._resource_name, self.service, task_name) self.start_timer(REMAINING_TIME_AFTER_STORE) self.actions_tracking.flush(self._timeout_event) if self.is_timed_out(): raise_exception( ERR_CREATING_TASKS_FOR_SELECTED_RESOURCES, task_name) self._timer.cancel() else: self.actions_tracking.flush() self._logger.info(INFO_ADDED_ITEMS, len(task_items), self.task[handlers.TASK_NAME]) running_time = float((datetime.now() - start).total_seconds()) self._logger.info(INFO_RESULT, running_time) if self.metrics: put_task_select_data(task_name=task_name, items=count_resource_items, selected_items=selected_resource_items, logger=self._logger, selection_time=running_time) return safe_dict({ "datetime": datetime.now().isoformat(), "running-time": running_time, "dispatched-tasks": task_items }) finally: self._logger.flush()
def handle_request(self): """ Handles the select resources request. Creates new actions for resources found for a task :return: Results of handling the request """ def is_selected_resource(resource, taskname, tags_filter, does_resource_supports_tags): """ Tests if item is a selected resource for this task :param resource: The tested resource :param taskname: Name of the task :param tags_filter: Tag filter :param does_resource_supports_tags: Trie if the resource supports tags :return: True if resource is selected """ # No tags then always selected if not does_resource_supports_tags: self._logger.debug(DEBUG_RESOURCE_NO_TAGS, resource) return True tags = resource.get("Tags", {}) # name of the tag that holds the list of tasks for this resource tagname = self._task_tag if tags_filter is None: # test if name of the task is in list of tasks in tag value if tagname in tags and taskname in [ x.strip() for x in tags[tagname].split(',') ]: self._logger.debug( DEBUG_SELECTED_BY_TASK_NAME_IN_TAG_VALUE, safe_json(resource, indent=2), tagname, taskname) return True else: # using a tag filter, * means any tag if tags_filter == "*": self._logger.debug(DEBUG_SELECTED_WILDCARD_TAG_FILTER, safe_json(resource, indent=2), taskname) return True # test if there are any tags matching the tag filter matched_tags = TagFilterSet( tags_filter).pairs_matching_any_filter(tags) if len(matched_tags) != 0: self._logger.debug(DEBUG_SELECTED_BY_TAG_FILTER, safe_json(resource, indent=2), matched_tags, tag_filter, taskname) return True self._logger.debug(DEBUG_RESOURCE_NOT_SELECTED, safe_json(resource, indent=2), taskname) return False def resource_batches(resources): """ Returns resources as chunks of size items. If the class has an optional custom aggregation function then the reousrces are aggregated first using this function before applying the batch size :param resources: resources to process :return: Generator for blocks of resource items """ aggregate_func = getattr(self.action_class, actions.CUSTOM_AGGREGATE_METHOD, None) batch_size = self.action_properties.get(actions.ACTION_BATCH_SIZE) for i in aggregate_func( resources, self.task_parameters) if aggregate_func is not None else [ resources ]: if batch_size is None: yield i else: first = 0 while first < len(i): yield i[first:first + batch_size] first += batch_size try: items = [] start = datetime.now() self._logger.info("Handler {}", self.__class__.__name__) self._logger.debug(DEBUG_EVENT, safe_json(self._event, indent=2)) self._logger.debug(DEBUG_ACTION, safe_json(self.action_properties, indent=2)) self._logger.info(INFO_SELECTED_RESOURCES, self.resource_name, self.service, self.task[handlers.TASK_NAME]) self._logger.info(INFO_AGGR_LEVEL, self.aggregation_level) task_level_aggregated_resources = [] args = self._build_describe_argument() supports_tags = self.action_properties.get( actions.ACTION_RESOURCES) in services.create_service( self.service).resources_with_tags args["tags"] = supports_tags self._logger.info(INFO_USE_TAGS_TO_SELECT, "R" if supports_tags else "No r") task_name = self.task[handlers.TASK_NAME] # get optional tag filter tag_filter = self.task.get(handlers.TASK_TAG_FILTER) if tag_filter is None: self._logger.debug(DEBUG_SELECT_BY_TASK_NAME, self.resource_name, self._task_tag, task_name) elif tag_filter == "*": self._logger.debug(DEBUG_SELECT_ALL_RESOURCES, self.resource_name) else: self._logger.debug(DEBUG_TAG_FILTER_USED_TO_SELECT_RESOURCES, self.resource_name) with TaskTrackingTable(self._context) as actions_tracking: for service in self._account_service_sessions(self.service): assumed_role = service.assumed_role self._logger.info(INFO_ACCOUNT, service.aws_account) if assumed_role is not None: self._logger.info(INFO_ASSUMED_ROLE, assumed_role) for region in self._regions: if region is not None: args["region"] = region else: if "region" in args: del args["region"] self._logger.debug(DEBUG_SELECT_PARAMETERS, self.resource_name, self.service, args) # selecting a list of all resources in this account/region all_resources = list( service.describe(self.resource_name, **args)) logstr = INFO_RESOURCES_FOUND.format( len(all_resources)) if region is not None: logstr = INFO_IN_REGION.format(logstr, region) self._logger.info(logstr) # select resources that are processed by the task selected = list([ sr for sr in all_resources if is_selected_resource( sr, task_name, tag_filter, supports_tags) ]) if len(all_resources) > 0: self._logger.info(INFO_RESOURCES_SELECTED, len(selected)) if len(selected) == 0: continue if not self.keep_tags: for res in selected: if "Tags" in res: del res["Tags"] if self.aggregation_level == actions.ACTION_AGGREGATION_TASK: task_level_aggregated_resources += selected elif self.aggregation_level == actions.ACTION_AGGREGATION_ACCOUNT: if self._check_can_execute(selected): # create tasks action for account aggregated resources , optionally split in batch size chunks for r in resource_batches(selected): action_item = actions_tracking.add_task_action( task=self.task, assumed_role=assumed_role, action_resources=r, task_datetime=self.task_dt, source=self.source) items.append(action_item) self._logger.info( INFO_ACCOUNT_AGGREGATED, action_item[tracking.TASK_TR_ID], len(r), self.resource_name, self.task[handlers.TASK_NAME]) else: for res in selected: # task action for each selected resource action_item = actions_tracking.add_task_action( task=self.task, assumed_role=assumed_role, action_resources=res, task_datetime=self.task_dt, source=self.source) items.append(action_item) self._logger.info( INFO_RESOURCE, action_item[tracking.TASK_TR_ID], self.resource_name, self.task[handlers.TASK_NAME]) if self.aggregation_level == actions.ACTION_AGGREGATION_TASK and len( task_level_aggregated_resources) > 0: if self._check_can_execute( task_level_aggregated_resources): for r in resource_batches( task_level_aggregated_resources): # create tasks action for task aggregated resources , optionally split in batch size chunks action_item = actions_tracking.add_task_action( task=self.task, assumed_role=None, action_resources=r, task_datetime=self.task_dt, source=self.source) items.append(action_item) self._logger.info(INFO_TASK_AGGREGATED, action_item[tracking.TASK_TR_ID], len(r), self.resource_name, self.task[handlers.TASK_NAME]) self._logger.info(INFO_ADDED_ITEMS, len(items), self.task[handlers.TASK_NAME]) running_time = float((datetime.now() - start).total_seconds()) self._logger.info(INFO_RESULT, running_time) return safe_dict({ "datetime": datetime.now().isoformat(), "running-time": running_time, "dispatched-tasks": items }) finally: self._logger.flush()
def run_reset(): def purge_table(table_name, key_name): db = get_client_with_retries( "dynamodb", ["scan", "delete_item", "batch_write_item"], session=boto3.Session()) def keys_to_delete(): key_ids = [] delete_args = { "TableName": table_name, "Select": "SPECIFIC_ATTRIBUTES", "ConsistentRead": True, "AttributesToGet": [key_name] } while True: resp = db.scan_with_retries(**delete_args) for item in resp.get("Items", []): key_ids.append(item[key_name]["S"]) delete_args["ExclusiveStartKey"] = resp.get( "LastEvaluatedKey", None) if delete_args["ExclusiveStartKey"] is None: break return key_ids keys = keys_to_delete() print("{} items to delete".format(len(keys))) # delete items in batches of max 25 items while True: delete_requests = [] while len(keys) > 0: delete_requests.append( {'DeleteRequest': { 'Key': { key_name: { "S": keys.pop(0) } } }}) if len(keys) == 0 or len(delete_requests) == 25: db.batch_write_item_with_retries( RequestItems={table_name: delete_requests}) print(".", end="") time.sleep(1) delete_requests = [] time.sleep(1) keys = keys_to_delete() if len(keys) == 0: break print("") streams = create_service( "Cloudwatchlogs", session=boto3.Session()).describe( "LogStreams", logGroupName=os.getenv(outputs.queued_logger.ENV_LOG_GROUP)) print("Purging table {}".format(handlers.ENV_ACTION_TRACKING_TABLE)) purge_table(os.getenv(handlers.ENV_ACTION_TRACKING_TABLE), handlers.TASK_TR_ID) print("Purging table {}".format(handlers.ENV_CONCURRENCY_TABLE)) purge_table(os.getenv(handlers.ENV_CONCURRENCY_TABLE), handlers.TASK_TR_CONCURRENCY_ID) cwl = get_client_with_retries("logs", ["delete_log_stream"], session=boto3.Session()) for stream in streams: print("Deleting logstream {}".format(stream["logStreamName"])) cwl.delete_log_stream_with_retries( logGroupName=os.getenv(outputs.queued_logger.ENV_LOG_GROUP), logStreamName=stream["logStreamName"])
def is_completed(self, snapshot_create_data): def grant_create_volume_permissions(snap_ids): if self.accounts_with_create_permissions is not None and len(self.accounts_with_create_permissions) > 0: args = { "CreateVolumePermission": { "Add": [{"UserId": a.strip()} for a in self.accounts_with_create_permissions] } } for snapshot_id in snap_ids: args["SnapshotId"] = snapshot_id try: self.ec2_client.modify_snapshot_attribute_with_retries(**args) self._logger_.info(INFO_SETTING_CREATE_VOLUME_PERMISSIONS, ", ".join(self.accounts_with_create_permissions)) self.result["create-volume-access-accounts"] = [a.strip() for a in self.accounts_with_create_permissions] except Exception as ex: raise_exception(ERR_SETTING_CREATE_VOLUME_PERMISSIONS, self.accounts_with_create_permissions, ex) def tag_shared_snapshots(snapshot_data, snap_ids): if self.accounts_with_create_permissions not in ["", None] and self.tag_shared_snapshots: for account in self.accounts_with_create_permissions: session_for_tagging = self.get_action_session(account=account, param_name=PARAM_SHARED_ACCOUNT_TAGGING_ROLENAME, logger=self._logger_) if session_for_tagging is None: self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account) continue try: ec2_client = get_client_with_retries(service_name="ec2", methods=[ "create_tags", "delete_tags" ], context=self._context_, region=self._region_, session=session_for_tagging, logger=self._logger_) for snap_id in snap_ids: tags = snapshot_data.get(snap_id, {}).get("tags", None) if tags is not None: self._logger_.info(INFO_SET_SNAPSHOT_TAGS_SHARED, safe_json(tags, indent=3), snap_id, account, self._region_) tagging.set_ec2_tags(ec2_client=ec2_client, resource_ids=[snap_id], tags=tags, logger=self._logger_) except Exception as ex: raise Exception(ERR_SETTING_SHARED_TAGS.format(account, str(ex))) def set_volume_tags(volume_id, snap_id): tags = self.build_tags_from_template(parameter_name=PARAM_VOLUME_TAGS, tag_variables={ TAG_PLACEHOLDER_VOLUME_SNAPSHOT: snap_id }) if len(tags) > 0: try: tagging.set_ec2_tags(ec2_client=self.ec2_client, resource_ids=[volume_id], tags=tags, logger=self._logger_) self._logger_.info(INFO_SET_VOLUME_TAGS, safe_json(tags, indent=3), volume_id) except Exception as ex: raise Exception(ERR_SETTING_VOLUME_TAGS.format(self.instance_id, ex)) def set_instance_tags(snap_ids): tags = self.build_tags_from_template(parameter_name=PARAM_INSTANCE_TAGS, tag_variables={ TAG_PLACEHOLDER_INSTANCE_SNAPSHOTS: ','.join(sorted(snap_ids)) }) if len(tags) > 0: try: self.set_ec2_instance_tags_with_event_loop_check(instance_ids=[self.instance_id], tags_to_set=tags, client=self.ec2_client, region=self._region_) self._logger_.info(INFO_SET_INSTANCE_TAGS, safe_json(tags, indent=3), self.instance_id) except Exception as ex: raise Exception(ERR_SETTING_INSTANCE_TAGS.format(self.instance_id, ex)) snapshot_ids = [volume.get("create_snapshot", {}).get("SnapshotId") for volume in list(snapshot_create_data.get("volumes", {}).values())] self._logger_.info(INFO_CHECKING_SNAPSHOT_STATUS, ",".join(snapshot_ids)) if len(snapshot_ids) == 0: return { "InstanceId": snapshot_create_data["instance"], "Volumes": [] } # create service instance to test is snapshots are available ec2 = services.create_service("ec2", session=self._session_, service_retry_strategy=get_default_retry_strategy("ec2", context=self._context_)) # test if the snapshot with the ids that were returned from the CreateSnapshot API call exists and are completed snapshots = list(ec2.describe(services.ec2_service.SNAPSHOTS, OwnerIds=["self"], region=self.instance["Region"], Filters=[ { "Name": "snapshot-id", "Values": snapshot_ids } ])) if len(snapshots) != len(snapshot_ids): # allow 5 minutes to all snapshots to appear start_time = dateutil.parser.parse(snapshot_create_data["start-time"]) if self._datetime_.now() - start_time < timedelta(minutes=5): self._logger_.info(INFO_NOT_ALL_IN_PROGRESS) return None test_result = { "InstanceId": snapshot_create_data["instance"], "Volumes": [{ "VolumeId": s["VolumeId"], "SnapshotId": s["SnapshotId"], "State": s["State"], "Progress": s["Progress"] } for s in snapshots] } self._logger_.info(INFO_STATE_SNAPSHOTS, safe_json(test_result, indent=3)) # wait until all snapshot are no longer pending for volume in test_result["Volumes"]: if volume["State"] == SNAPSHOT_STATE_PENDING: self._logger_.info(INFO_CREATION_PENDING) return None # collect possible failed snapshots failed = [] for volume in test_result["Volumes"]: if volume["State"] == SNAPSHOT_STATE_ERROR: failed.append(volume) if len(failed) > 0: s = ",".join([ERR_FAILED_SNAPSHOT.format(volume["SnapshotId"], volume["VolumeId"]) for volume in failed]) raise Exception(s) if len(snapshot_ids) != len(snapshots): created_snapshots = [s["SnapshotId"] for s in snapshots] raise Exception(ERR_MISSING_SNAPSHOTS.format(",".join([s for s in snapshot_ids if s not in created_snapshots]))) snapshot_ids = [s["SnapshotId"] for s in snapshots] # set tags on source instance set_instance_tags(snapshot_ids) for s in snapshots: set_volume_tags(volume_id=s["VolumeId"], snap_id=s["SnapshotId"]) # set permissions to create volumes from snapshots grant_create_volume_permissions(snapshot_ids) # tag resources in accounts the snapshots are shared with tag_shared_snapshots(snapshot_create_data.get("snapshots", {}), snapshot_ids) self._logger_.info(INFO_COMPLETED) return test_result