def _add_cfn_parameters(self):
if (self.config.dev_settings and self.config.dev_settings.cookbook
and self.config.dev_settings.cookbook.chef_cookbook):
dev_settings_cookbook_value = self.config.dev_settings.cookbook.chef_cookbook
custom_chef_cookbook = (
create_s3_presigned_url(dev_settings_cookbook_value)
if dev_settings_cookbook_value.startswith("s3://") else
dev_settings_cookbook_value)
else:
custom_chef_cookbook = ""
CfnParameter(
self,
"CfnParamCookbookVersion",
type="String",
default=utils.get_installed_version(),
description="CookbookVersion",
)
CfnParameter(self,
"CfnParamChefCookbook",
type="String",
default=custom_chef_cookbook,
description="ChefCookbook")
CfnParameter(self,
"CfnParamCincInstaller",
type="String",
default="",
description="CincInstaller")
CfnParameter(
self,
"CfnParamChefDnaJson",
type="String",
default=ImageBuilderExtraChefAttributes(
self.config.dev_settings).dump_json(),
description="ChefAttributes",
)
CfnParameter(
self,
"CfnParamUpdateOsAndReboot",
type="String",
default="true"
if self.config.build and self.config.build.update_os_packages
and self.config.build.update_os_packages.enabled else "false",
description="UpdateOsAndReboot",
)
def export_logs(
self,
bucket: str,
bucket_prefix: str = None,
keep_s3_objects: bool = False,
start_time: datetime = None,
end_time: datetime = None,
output_file: str = None,
):
"""
Export image builder's logs in the given output path, by using given bucket as a temporary folder.
:param bucket: S3 bucket to be used to export cluster logs data
:param bucket_prefix: Key path under which exported logs data will be stored in s3 bucket,
also serves as top-level directory in resulting archive
:param keep_s3_objects: Keep the exported objects exports to S3. The default behavior is to delete them
:param start_time: Start time of interval of interest for log events. ISO 8601 format: YYYY-MM-DDThh:mm:ssTZD
:param end_time: End time of interval of interest for log events. ISO 8601 format: YYYY-MM-DDThh:mm:ssTZD
"""
# check stack
stack_exists = self._stack_exists()
if not stack_exists:
LOGGER.debug("CloudFormation Stack for Image %s does not exist.", self.image_id)
try:
with tempfile.TemporaryDirectory() as output_tempdir:
# Create root folder for the archive
archive_name = f"{self.image_id}-logs-{datetime.now().strftime('%Y%m%d%H%M')}"
root_archive_dir = os.path.join(output_tempdir, archive_name)
os.makedirs(root_archive_dir, exist_ok=True)
if AWSApi.instance().logs.log_group_exists(self._log_group_name):
# Export logs from CloudWatch
export_logs_filters = self._init_export_logs_filters(start_time, end_time)
logs_exporter = CloudWatchLogsExporter(
resource_id=self.image_id,
log_group_name=self._log_group_name,
bucket=bucket,
output_dir=root_archive_dir,
bucket_prefix=bucket_prefix,
keep_s3_objects=keep_s3_objects,
)
logs_exporter.execute(
start_time=export_logs_filters.start_time, end_time=export_logs_filters.end_time
)
else:
LOGGER.info(
"Log streams not yet available for %s, only CFN Stack events will be exported.", {self.image_id}
)
if stack_exists:
# Get stack events and write them into a file
stack_events_file = os.path.join(root_archive_dir, self._stack_events_stream_name)
export_stack_events(self.stack.name, stack_events_file)
archive_path = create_logs_archive(root_archive_dir, output_file)
if output_file:
return output_file
else:
s3_path = upload_archive(bucket, bucket_prefix, archive_path)
return create_s3_presigned_url(s3_path)
except Exception as e:
raise ImageBuilderActionError(f"Unexpected error when exporting image's logs: {e}")
def export_logs(
self,
bucket: str,
bucket_prefix: str = None,
keep_s3_objects: bool = False,
start_time: datetime = None,
end_time: datetime = None,
filters: str = None,
output_file: str = None,
):
"""
Export cluster's logs in the given output path, by using given bucket as a temporary folder.
:param output: file path to save log file archive to
:param bucket: Temporary S3 bucket to be used to export cluster logs data
:param bucket_prefix: Key path under which exported logs data will be stored in s3 bucket,
also serves as top-level directory in resulting archive
:param keep_s3_objects: Keep the exported objects exports to S3. The default behavior is to delete them
:param start_time: Start time of interval of interest for log events. ISO 8601 format: YYYY-MM-DDThh:mm:ssTZD
:param end_time: End time of interval of interest for log events. ISO 8601 format: YYYY-MM-DDThh:mm:ssTZD
:param filters: Filters in the format Name=name,Values=value1,value2
Accepted filters are: private_dns_name, node_type==HeadNode
"""
# check stack
if not AWSApi.instance().cfn.stack_exists(self.stack_name):
raise NotFoundClusterActionError(
f"Cluster {self.name} does not exist.")
try:
with tempfile.TemporaryDirectory() as output_tempdir:
# Create root folder for the archive
archive_name = f"{self.name}-logs-{datetime.now().strftime('%Y%m%d%H%M')}"
root_archive_dir = os.path.join(output_tempdir, archive_name)
os.makedirs(root_archive_dir, exist_ok=True)
if self.stack.log_group_name:
# Export logs from CloudWatch
export_logs_filters = self._init_export_logs_filters(
start_time, end_time, filters)
logs_exporter = CloudWatchLogsExporter(
resource_id=self.name,
log_group_name=self.stack.log_group_name,
bucket=bucket,
output_dir=root_archive_dir,
bucket_prefix=bucket_prefix,
keep_s3_objects=keep_s3_objects,
)
logs_exporter.execute(
log_stream_prefix=export_logs_filters.
log_stream_prefix,
start_time=export_logs_filters.start_time,
end_time=export_logs_filters.end_time,
)
else:
LOGGER.debug(
"CloudWatch logging is not enabled for cluster %s, only CFN Stack events will be exported.",
{self.name},
)
# Get stack events and write them into a file
stack_events_file = os.path.join(
root_archive_dir, self._stack_events_stream_name)
export_stack_events(self.stack_name, stack_events_file)
archive_path = create_logs_archive(root_archive_dir,
output_file)
if output_file:
return output_file
else:
s3_path = upload_archive(bucket, bucket_prefix,
archive_path)
return create_s3_presigned_url(s3_path)
except Exception as e:
raise ClusterActionError(
f"Unexpected error when exporting cluster's logs: {e}")