def stream_instance_logs_from_cloudwatch(
sleep_time=10,
log_group=None,
specific_log_stream=None
):
"""
Method streams CloudWatch logs to the terminal for the logGroup given. Since it is possible that the logGroup
might match multiple logGroups, multiple threads can be spawned to switch between streams to display all of them
on the same terminal.
:param sleep_time: sleep time to refresh the logs from cloudwatch
:param log_group: cloudwatch logGroup
:param specific_log_stream: since all of our log streams are instance ids we require this if we want a single stream
"""
streamer = io.get_event_streamer()
streamer.prompt = ' -- {0} -- (Ctrl+C to exit)'.format(log_group)
start_time = None
while True:
log_group_names = set(cloudwatch_log_stream_names(log_group, specific_log_stream))
for log_group_name in log_group_names:
_create_log_stream_for_log_group(log_group, log_group_name, streamer, sleep_time, start_time)
_delay_subsequent_stream_creation()
_wait_to_poll_cloudwatch()
start_time = _updated_start_time()
def stream_instance_logs_from_cloudwatch(
sleep_time=10,
log_group=None,
specific_log_stream=None
):
"""
Method streams CloudWatch logs to the terminal for the logGroup given. Since it is possible that the logGroup
might match multiple logGroups, multiple threads can be spawned to switch between streams to display all of them
on the same terminal.
:param sleep_time: sleep time to refresh the logs from cloudwatch
:param log_group: cloudwatch logGroup
:param specific_log_stream: since all of our log streams are instance ids we require this if we want a single stream
"""
streamer = io.get_event_streamer()
streamer.prompt = ' -- {0} -- (Ctrl+C to exit)'.format(log_group)
start_time = None
while True:
log_group_names = set(cloudwatch_log_stream_names(log_group, specific_log_stream))
for log_group_name in log_group_names:
_create_log_stream_for_log_group(log_group, log_group_name, streamer, sleep_time, start_time)
_delay_subsequent_stream_creation()
_wait_to_poll_cloudwatch()
start_time = _updated_start_time()
def test_event_streamer_with_unsafe_exit(self, mock_sys):
mock_sys.stdout.isatty.return_value = True
with mock.patch('ebcli.core.io.echo') as echo_mocked:
streamer = io.get_event_streamer()
streamer.stream_event("msg1", safe_to_quit=False)
echo_mocked.assert_called_with(streamer.unsafe_prompt, end='')
streamer.end_stream()
self.assertEqual(streamer.eventcount, 1, "Expected event count to be 1 but was: {0}".format(streamer.eventcount))
def test_pipe_streamer(self, mock_sys):
mock_sys.stdout.isatty.return_value = False
message = "msg1"
with mock.patch('ebcli.core.io.echo') as echo_mocked:
streamer = io.get_event_streamer()
streamer.stream_event(message)
echo_mocked.assert_called_with(message)
streamer.end_stream()
self.assertEqual(streamer.eventcount, 0, "Expected event count to be 0 but was: {0}".format(streamer.eventcount))
def test_event_streamer(self, mock_sys):
mock_sys.stdout.isatty.return_value = True
with mock.patch('ebcli.core.io.echo') as echo_mocked:
streamer = io.get_event_streamer()
streamer.stream_event("msg1")
streamer.stream_event("msg2")
echo_mocked.assert_called_with(streamer.prompt, end='')
streamer.end_stream()
self.assertEqual(streamer.eventcount, 2, "Expected event count to be 2 but was: {0}".format(streamer.eventcount))
def stream_platform_logs(platform_name, version, streamer=None, sleep_time=4, log_name=None, formatter=None):
log_group_name = _get_platform_builder_group_name(platform_name)
if streamer is None:
streamer = io.get_event_streamer()
if log_name is not None:
streamer.prompt = ' -- Streaming logs for %s -- (Ctrl+C to exit)' % log_name
stream_single_stream(log_group_name, version, streamer, sleep_time, formatter)
def stream_platform_logs(response, platform_name, version, timeout):
arn = response['PlatformSummary']['PlatformArn']
request_id = response['ResponseMetadata']['RequestId']
streamer = io.get_event_streamer()
builder_events = threading.Thread(target=logsops.stream_platform_logs,
args=(platform_name, version, streamer,
5, None, PackerStreamFormatter()))
builder_events.daemon = True
builder_events.start()
commonops.wait_for_success_events(request_id,
platform_arn=arn,
streamer=streamer,
timeout_in_minutes=timeout or 30)
def stream_platform_logs(platform_name, version, streamer=None, sleep_time=4, log_name=None, formatter=None):
"""
Streams the logs of a custom platform
:param platform_name: A CloudWatch logGroup in which the logStream `version` exists
:param version: A CloudWatch logStream to poll
:param streamer: The object that streams events to the terminal
:param sleep_time: Time in seconds to sleep before polling CloudWatch for newer events
:param log_name: A name used to identify the blob of output in the terminal for the logStream
:param formatter: The object that formats the output to be displayed in the terminal
:return: None
"""
log_group_name = _get_platform_builder_group_name(platform_name)
wait_for_log_group_to_come_into_existence(log_group_name, sleep_time)
streamer = streamer or io.get_event_streamer()
if log_name:
streamer.prompt = ' -- Streaming logs for %s -- (Ctrl+C to exit)' % log_name
stream_single_stream(log_group_name, version, sleep_time, None, formatter)
def stream_platform_logs(platform_name, version, streamer=None, sleep_time=4, log_name=None, formatter=None):
"""
Streams the logs of a custom platform
:param platform_name: A CloudWatch logGroup in which the logStream `version` exists
:param version: A CloudWatch logStream to poll
:param streamer: The object that streams events to the terminal
:param sleep_time: Time in seconds to sleep before polling CloudWatch for newer events
:param log_name: A name used to identify the blob of output in the terminal for the logStream
:param formatter: The object that formats the output to be displayed in the terminal
:return: None
"""
log_group_name = _get_platform_builder_group_name(platform_name)
wait_for_log_group_to_come_into_existence(log_group_name, sleep_time)
streamer = streamer or io.get_event_streamer()
if log_name:
streamer.prompt = ' -- Streaming logs for %s -- (Ctrl+C to exit)' % log_name
stream_single_stream(log_group_name, version, sleep_time, None, formatter)
def stream_cloudwatch_logs(env_name,
sleep_time=2,
log_group=None,
instance_id=None):
"""
This function will stream logs to the terminal for the log group given, if multiple streams are found we will
spawn multiple threads with each stream switch between them to stream all streams at the same time.
:param env_name: environment name
:param sleep_time: sleep time to refresh the logs from cloudwatch
:param log_group: cloudwatch log group
:param instance_id: since all of our log streams are instance ids we require this if we want a single stream
"""
if log_group is None:
log_group = 'awseb-{0}-activity'.format(env_name)
log_name = 'eb-activity.log'
else:
log_name = get_log_name(log_group)
stream_names = []
streamer = io.get_event_streamer()
streamer.prompt = ' -- {0} -- (Ctrl+C to exit)'.format(log_name)
jobs = []
while True:
try:
new_names = cloudwatch.get_all_stream_names(log_group, instance_id)
except:
raise NotFoundError(strings['cloudwatch-stream.notsetup'])
if len(new_names) == 0:
raise NotFoundError(strings['cloudwatch-logs.nostreams'].replace(
'{log_group}', log_group))
for name in new_names:
if name not in stream_names:
stream_names.append(name)
p = threading.Thread(target=_stream_single_stream,
args=(log_group, name, streamer,
sleep_time))
p.daemon = True
jobs.append(p)
p.start()
time.sleep(0.2) # offset threads
time.sleep(10)
def follow_events(app_name, env_name, platform_arn=None):
last_time = None
streamer = io.get_event_streamer()
try:
while True:
events = elasticbeanstalk.get_new_events(app_name,
env_name,
None,
platform_arn=platform_arn,
last_event_time=last_time)
for event in reversed(events):
message = commonops.get_event_string(event, long_format=True)
streamer.stream_event(message)
last_time = event.event_date
_sleep()
except EndOfTestError:
pass
finally:
streamer.end_stream()
def create_platform_version(
version,
major_increment,
minor_increment,
patch_increment,
instance_type,
vpc = None,
staged=False,
timeout=None):
platform_name = fileoperations.get_platform_name()
instance_profile = fileoperations.get_instance_profile(None)
key_name = commonops.get_default_keyname()
if version is None:
version = _get_latest_version(platform_name=platform_name, owner=Constants.OWNED_BY_SELF, ignored_states=[])
if version is None:
version = '1.0.0'
else:
major, minor, patch = version.split('.', 3)
if major_increment:
major = str(int(major) + 1)
minor = '0'
patch = '0'
if minor_increment:
minor = str(int(minor) + 1)
patch = '0'
if patch_increment or not(major_increment or minor_increment):
patch = str(int(patch) + 1)
version = "%s.%s.%s" % (major, minor, patch)
if not VALID_PLATFORM_VERSION_FORMAT.match(version):
raise InvalidPlatformVersionError(strings['exit.invalidversion'])
cwd = os.getcwd()
fileoperations._traverse_to_project_root()
try:
if heuristics.directory_is_empty():
raise PlatformWorkspaceEmptyError(strings['exit.platformworkspaceempty'])
finally:
os.chdir(cwd)
if not heuristics.has_platform_definition_file():
raise PlatformWorkspaceEmptyError(strings['exit.no_pdf_file'])
source_control = SourceControl.get_source_control()
if source_control.untracked_changes_exist():
io.log_warning(strings['sc.unstagedchanges'])
version_label = source_control.get_version_label()
if staged:
# Make a unique version label
timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
version_label = version_label + '-stage-' + timestamp
file_descriptor, original_platform_yaml = tempfile.mkstemp()
os.close(file_descriptor)
copyfile('platform.yaml', original_platform_yaml)
s3_bucket = None
s3_key = None
try:
# Add option settings to platform.yaml
_enable_healthd()
s3_bucket, s3_key = get_app_version_s3_location(platform_name, version_label)
# Create zip file if the application version doesn't exist
if s3_bucket is None and s3_key is None:
file_name, file_path = _zip_up_project(version_label, source_control, staged=staged)
else:
file_name = None
file_path = None
finally:
# Restore original platform.yaml
move(original_platform_yaml, 'platform.yaml')
# Use existing bucket if it exists
bucket = elasticbeanstalk.get_storage_location() if s3_bucket is None else s3_bucket
# Use existing key if it exists
key = platform_name + '/' + file_name if s3_key is None else s3_key
try:
s3.get_object_info(bucket, key)
io.log_info('S3 Object already exists. Skipping upload.')
except NotFoundError:
io.log_info('Uploading archive to s3 location: ' + key)
s3.upload_platform_version(bucket, key, file_path)
# Just deletes the local zip
fileoperations.delete_app_versions()
io.log_info('Creating Platform Version ' + version_label)
response = elasticbeanstalk.create_platform_version(
platform_name, version, bucket, key, instance_profile, key_name, instance_type, vpc)
# TODO: Enable this once the API returns the name of the environment associated with a
# CreatePlatformRequest, and remove hard coded value. There is currently only one type
# of platform builder, we may support additional builders in the future.
#environment_name = response['PlatformSummary']['EnvironmentName']
environment_name = 'eb-custom-platform-builder-packer'
io.echo(colored(
strings['platformbuildercreation.info'].format(environment_name), attrs=['reverse']))
fileoperations.update_platform_version(version)
commonops.set_environment_for_current_branch(environment_name)
arn = response['PlatformSummary']['PlatformArn']
request_id = response['ResponseMetadata']['RequestId']
if not timeout:
timeout = 30
# Share streamer for platform events and builder events
streamer = io.get_event_streamer()
builder_events = threading.Thread(
target=logsops.stream_platform_logs,
args=(platform_name, version, streamer, 5, None, PackerStreamFormatter()))
builder_events.daemon = True
# Watch events from builder logs
builder_events.start()
commonops.wait_for_success_events(
request_id,
platform_arn=arn,
streamer=streamer,
timeout_in_minutes=timeout
)
def wait_for_success_events(request_id, timeout_in_minutes=None,
sleep_time=5, stream_events=True, can_abort=False,
streamer=None, app_name=None, env_name=None, version_label=None,
platform_arn=None, timeout_error_message=None):
if timeout_in_minutes == 0:
return
if timeout_in_minutes is None:
timeout_in_minutes = 10
start = datetime.utcnow()
timediff = timedelta(seconds=timeout_in_minutes * 60)
last_time = start
if streamer is None:
streamer = io.get_event_streamer()
if can_abort:
streamer.prompt += strings['events.abortmessage']
events = []
safe_to_quit = True
if version_label is not None and request_id is None:
safe_to_quit = False
try:
if request_id:
while not events:
events = elasticbeanstalk.get_new_events(
app_name,
env_name,
request_id,
last_event_time=None,
platform_arn=platform_arn,
version_label=version_label
)
if len(events) > 0:
event = events[-1]
app_name = event.app_name
env_name = event.environment_name
if stream_events:
streamer.stream_event(
get_event_string(
event,
long_format=True
),
safe_to_quit=safe_to_quit
)
_raise_if_error_event(event.message)
if _is_success_event(event.message):
return
last_time = event.event_date
else:
_sleep(sleep_time)
while not _timeout_reached(start, timediff):
_sleep(sleep_time)
events = elasticbeanstalk.get_new_events(
app_name,
env_name,
request_id,
last_event_time=last_time,
platform_arn=platform_arn,
version_label=version_label
)
if events:
events = filter_events(
events,
env_name=env_name,
request_id=request_id,
version_label=version_label
)
for event in reversed(events):
if stream_events:
streamer.stream_event(
get_event_string(
event,
long_format=True
),
safe_to_quit=safe_to_quit
)
last_time = event.event_date
_raise_if_error_event(event.message)
if _is_success_event(event.message):
return
finally:
streamer.end_stream()
if not timeout_error_message:
timeout_error_message = strings['timeout.error'].format(timeout_in_minutes=timeout_in_minutes)
raise TimeoutError(timeout_error_message)
def wait_for_compose_events(request_id, app_name, grouped_envs, timeout_in_minutes=None,
sleep_time=5, stream_events=True,
can_abort=False):
if timeout_in_minutes == 0:
return
if timeout_in_minutes is None:
timeout_in_minutes = 15
start = datetime.utcnow()
timediff = timedelta(seconds=timeout_in_minutes * 60)
last_times = []
events_matrix = []
successes = []
last_time_compose = datetime.utcnow()
compose_events = []
for i in range(len(grouped_envs)):
last_times.append(datetime.utcnow())
events_matrix.append([])
successes.append(False)
streamer = io.get_event_streamer()
if can_abort:
streamer.prompt += strings['events.abortmessage']
try:
while not _timeout_reached(start, timediff):
if all(successes):
return
compose_events = elasticbeanstalk.get_new_events(app_name=app_name,
env_name=None,
request_id=request_id,
last_event_time=last_time_compose)
for event in reversed(compose_events):
if stream_events:
streamer.stream_event(get_compose_event_string(event))
last_time_compose = event.event_date
for index in range(len(grouped_envs)):
if successes[index]:
continue
_sleep(sleep_time)
events_matrix[index] = elasticbeanstalk.get_new_events(
app_name, grouped_envs[index], None,
last_event_time=last_times[index]
)
for event in reversed(events_matrix[index]):
if stream_events:
streamer.stream_event(get_env_event_string(event))
last_times[index] = event.event_date
if _is_success_event(event.message):
successes[index] = True
finally:
streamer.end_stream()
io.log_error(strings['timeout.error'])
def create_platform_version(version,
major_increment,
minor_increment,
patch_increment,
instance_type,
vpc=None,
staged=False,
timeout=None):
platform_name = fileoperations.get_platform_name()
instance_profile = fileoperations.get_instance_profile(None)
key_name = commonops.get_default_keyname()
if version is None:
version = _get_latest_version(platform_name=platform_name,
owner=Constants.OWNED_BY_SELF,
ignored_states=[])
if version is None:
version = '1.0.0'
else:
major, minor, patch = version.split('.', 3)
if major_increment:
major = str(int(major) + 1)
minor = '0'
patch = '0'
if minor_increment:
minor = str(int(minor) + 1)
patch = '0'
if patch_increment or not (major_increment or minor_increment):
patch = str(int(patch) + 1)
version = "%s.%s.%s" % (major, minor, patch)
if not VALID_PLATFORM_VERSION_FORMAT.match(version):
raise InvalidPlatformVersionError(strings['exit.invalidversion'])
cwd = os.getcwd()
fileoperations._traverse_to_project_root()
try:
if heuristics.directory_is_empty():
raise PlatformWorkspaceEmptyError(
strings['exit.platformworkspaceempty'])
finally:
os.chdir(cwd)
if not heuristics.has_platform_definition_file():
raise PlatformWorkspaceEmptyError(strings['exit.no_pdf_file'])
source_control = SourceControl.get_source_control()
if source_control.untracked_changes_exist():
io.log_warning(strings['sc.unstagedchanges'])
version_label = source_control.get_version_label()
if staged:
# Make a unique version label
timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
version_label = version_label + '-stage-' + timestamp
file_descriptor, original_platform_yaml = tempfile.mkstemp()
os.close(file_descriptor)
copyfile('platform.yaml', original_platform_yaml)
try:
# Add option settings to platform.yaml
_enable_healthd()
s3_bucket, s3_key = get_app_version_s3_location(
platform_name, version_label)
# Create zip file if the application version doesn't exist
if s3_bucket is None and s3_key is None:
file_name, file_path = _zip_up_project(version_label,
source_control,
staged=staged)
else:
file_name = None
file_path = None
finally:
# Restore original platform.yaml
move(original_platform_yaml, 'platform.yaml')
# Use existing bucket if it exists
bucket = elasticbeanstalk.get_storage_location(
) if s3_bucket is None else s3_bucket
# Use existing key if it exists
key = platform_name + '/' + file_name if s3_key is None else s3_key
try:
s3.get_object_info(bucket, key)
io.log_info('S3 Object already exists. Skipping upload.')
except NotFoundError:
io.log_info('Uploading archive to s3 location: ' + key)
s3.upload_platform_version(bucket, key, file_path)
# Just deletes the local zip
fileoperations.delete_app_versions()
io.log_info('Creating Platform Version ' + version_label)
response = elasticbeanstalk.create_platform_version(
platform_name, version, bucket, key, instance_profile, key_name,
instance_type, vpc)
# TODO: Enable this once the API returns the name of the environment associated with a
# CreatePlatformRequest, and remove hard coded value. There is currently only one type
# of platform builder, we may support additional builders in the future.
#environment_name = response['PlatformSummary']['EnvironmentName']
environment_name = 'eb-custom-platform-builder-packer'
io.echo(
colored(
strings['platformbuildercreation.info'].format(environment_name),
attrs=['reverse']))
fileoperations.update_platform_version(version)
commonops.set_environment_for_current_branch(environment_name)
arn = response['PlatformSummary']['PlatformArn']
request_id = response['ResponseMetadata']['RequestId']
if not timeout:
timeout = 30
# Share streamer for platform events and builder events
streamer = io.get_event_streamer()
builder_events = threading.Thread(target=logsops.stream_platform_logs,
args=(platform_name, version, streamer,
5, None, PackerStreamFormatter()))
builder_events.daemon = True
# Watch events from builder logs
builder_events.start()
commonops.wait_for_success_events(request_id,
platform_arn=arn,
streamer=streamer,
timeout_in_minutes=timeout)
def wait_for_success_events(request_id,
timeout_in_minutes=None,
sleep_time=5,
stream_events=True,
can_abort=False,
streamer=None,
app_name=None,
env_name=None,
version_label=None,
platform_arn=None,
timeout_error_message=None):
if timeout_in_minutes == 0:
return
if timeout_in_minutes is None:
timeout_in_minutes = 10
start = datetime.utcnow()
timediff = timedelta(seconds=timeout_in_minutes * 60)
# default to now, will update if request_id is provided
last_time = start
if streamer is None:
streamer = io.get_event_streamer()
if can_abort:
streamer.prompt += strings['events.abortmessage']
events = []
# If the even stream is terminated before we finish streaming application version events we will not
# be able to continue the command so we must warn the user it is not safe to quit.
safe_to_quit = True
if version_label is not None and request_id is None:
safe_to_quit = False
try:
# Get first event in order to get start time
if request_id:
while not events:
events = elasticbeanstalk.get_new_events(
app_name,
env_name,
request_id,
last_event_time=None,
platform_arn=platform_arn,
version_label=version_label)
if len(events) > 0:
event = events[-1]
app_name = event.app_name
env_name = event.environment_name
if stream_events:
streamer.stream_event(get_event_string(
event, long_format=True),
safe_to_quit=safe_to_quit)
_raise_if_error_event(event.message)
if _is_success_event(event.message):
return
last_time = event.event_date
else:
_sleep(sleep_time)
# Get remaining events
while not _timeout_reached(start, timediff):
_sleep(sleep_time)
events = elasticbeanstalk.get_new_events(
app_name,
env_name,
request_id,
last_event_time=last_time,
platform_arn=platform_arn,
version_label=version_label)
if events:
events = filter_events(events,
env_name=env_name,
request_id=request_id,
version_label=version_label)
for event in reversed(events):
if stream_events:
streamer.stream_event(get_event_string(event,
long_format=True),
safe_to_quit=safe_to_quit)
# We dont need to update last_time if we are not printing.
# This can solve timing issues
last_time = event.event_date
_raise_if_error_event(event.message)
if _is_success_event(event.message):
return
finally:
streamer.end_stream()
# We have timed out
if not timeout_error_message:
timeout_error_message = strings['timeout.error'].format(
timeout_in_minutes=timeout_in_minutes)
raise TimeoutError(timeout_error_message)
