def wait_for_scale(bundle_id, expected_scale, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
bundle_scale = get_scale(bundle_id, args)
if bundle_scale == expected_scale:
log.info('Bundle {} expected scale {} is met'.format(
bundle_id, expected_scale))
return
else:
log.info('Bundle {} waiting to reach expected scale {}'.format(
bundle_id, expected_scale))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(
bundle_events_url, headers=conduct_url.request_headers(args))
for event in sse_events:
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError(
'Bundle {} waiting to reach expected scale {}'.format(
bundle_id, expected_scale))
if event.event and event.event.startswith('bundleExecution'):
bundle_scale = get_scale(bundle_id, args)
if bundle_scale == expected_scale:
log.info('Bundle {} expected scale {} is met'.format(
bundle_id, expected_scale))
return
else:
log.info('Bundle {} has scale {}, expected {}'.format(
bundle_id, bundle_scale, expected_scale))
raise WaitTimeoutError(
'Bundle {} waiting to reach expected scale {}'.format(
bundle_id, expected_scale))
def wait_for_condition(bundle_id, condition, condition_name, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
installed_bundles = count_installations(bundle_id, args)
if condition(installed_bundles):
log.info('Bundle {} is {}'.format(bundle_id, condition_name))
return
else:
log.info('Bundle {} waiting to be {}'.format(bundle_id, condition_name))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(bundle_events_url)
for event in sse_events:
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError('Bundle {} waiting to be {}'.format(bundle_id, condition_name))
if event.event and event.event.startswith('bundleInstallation'):
installed_bundles = count_installations(bundle_id, args)
if condition(installed_bundles):
log.info('Bundle {} {}'.format(bundle_id, condition_name))
return
else:
log.info('Bundle {} still waiting to be {}'.format(bundle_id, condition_name))
raise WaitTimeoutError('Bundle {} still waiting to be {}'.format(bundle_id, condition_name))
def wait_for_scale(bundle_id, expected_scale, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
bundle_scale = get_scale(bundle_id, args)
if bundle_scale == expected_scale:
log.info('Bundle {} expected scale {} is met'.format(bundle_id, expected_scale))
return
else:
log.info('Bundle {} waiting to reach expected scale {}'.format(bundle_id, expected_scale))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(bundle_events_url)
for event in sse_events:
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError('Bundle {} waiting to reach expected scale {}'.format(bundle_id, expected_scale))
if event.event and event.event.startswith('bundleExecution'):
bundle_scale = get_scale(bundle_id, args)
if bundle_scale == expected_scale:
log.info('Bundle {} expected scale {} is met'.format(bundle_id, expected_scale))
return
else:
log.info('Bundle {} has scale {}, expected {}'.format(bundle_id, bundle_scale, expected_scale))
raise WaitTimeoutError('Bundle {} waiting to reach expected scale {}'.format(bundle_id, expected_scale))
def test_sse(self):
raw_sse = strip_margin("""|data:
|
|event:My Test Event
|data:My Test Data
|
|data:
|
|""")
raw_sse_iterators = [iter(list(raw_sse))]
iter_content_mock = MagicMock(side_effect=raw_sse_iterators)
raise_for_status_mock = MagicMock()
response_mock = MagicMock()
response_mock.raise_for_status = raise_for_status_mock
response_mock.iter_content = iter_content_mock
request_get_mock = MagicMock(return_value=response_mock)
result = []
with patch('requests.get', request_get_mock):
events = sse_client.get_events(False, '127.0.0.1', 'http://host.com')
for event in events:
result.append(event)
self.assertEqual([
sse_client.Event(event=None, data=''),
sse_client.Event(event='My Test Event', data='My Test Data'),
sse_client.Event(event=None, data='')
], result)
request_get_mock.assert_called_with('http://host.com', stream=True, **sse_client.SSE_REQUEST_INPUT)
raise_for_status_mock.assert_called_with()
iter_content_mock.assert_called_with(decode_unicode=True)
def wait_for_condition(bundle_id, condition, condition_name, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
installed_bundles = count_installations(bundle_id, args)
if condition(installed_bundles):
log.info('Bundle {} is {}'.format(bundle_id, condition_name))
return
else:
log.info('Bundle {} waiting to be {}'.format(bundle_id, condition_name))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(bundle_events_url, headers=conduct_url.request_headers(args))
for event in sse_events:
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError('Bundle {} waiting to be {}'.format(bundle_id, condition_name))
if event.event and event.event.startswith('bundleInstallation'):
installed_bundles = count_installations(bundle_id, args)
if condition(installed_bundles):
log.info('Bundle {} {}'.format(bundle_id, condition_name))
return
else:
log.info('Bundle {} still waiting to be {}'.format(bundle_id, condition_name))
raise WaitTimeoutError('Bundle {} still waiting to be {}'.format(bundle_id, condition_name))
def wait_for_condition(bundle_id, condition, condition_name, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
installed_bundles = count_installations(bundle_id, args)
last_log_message = None
if condition(installed_bundles):
log.info('Bundle {} is {}'.format(bundle_id, condition_name))
return
else:
sse_heartbeat_count_after_event = 0
log.info('Bundle {} waiting to be {}'.format(bundle_id,
condition_name))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(
args.dcos_mode,
conduct_url.conductr_host(args),
bundle_events_url,
auth=args.conductr_auth,
verify=args.server_verification_file)
for event in sse_events:
sse_heartbeat_count_after_event += 1
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError('Bundle {} waiting to be {}'.format(
bundle_id, condition_name))
# Check for installed bundles every 3 heartbeats from the last received event.
if event.event or (sse_heartbeat_count_after_event % 3 == 0):
if event.event:
sse_heartbeat_count_after_event = 0
installed_bundles = count_installations(bundle_id, args)
if condition(installed_bundles):
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
log.info('Bundle {} {}'.format(bundle_id, condition_name))
return
else:
if last_log_message:
last_log_message = '{}.'.format(last_log_message)
else:
last_log_message = 'Bundle {} still waiting to be {}'.format(
bundle_id, condition_name)
log.progress(last_log_message, flush=False)
raise WaitTimeoutError('Bundle {} waiting to be {}'.format(
bundle_id, condition_name))
def wait_for_scale(bundle_id, expected_scale, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
bundle_scale = get_scale(bundle_id, args)
if bundle_scale == expected_scale:
log.info('Bundle {} expected scale {} is met'.format(bundle_id, expected_scale))
return
else:
sse_heartbeat_count_after_event = 0
log.info('Bundle {} waiting to reach expected scale {}'.format(bundle_id, expected_scale))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(args.dcos_mode, conduct_url.conductr_host(args), bundle_events_url,
auth=args.conductr_auth, verify=args.server_verification_file)
last_scale = -1
last_log_message = None
for event in sse_events:
sse_heartbeat_count_after_event += 1
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError('Bundle {} waiting to reach expected scale {}'.format(bundle_id, expected_scale))
# Check for bundle scale every 3 heartbeats from the last received event.
if event.event or (sse_heartbeat_count_after_event % 3 == 0):
if event.event:
sse_heartbeat_count_after_event = 0
bundle_scale = get_scale(bundle_id, args)
if bundle_scale == expected_scale:
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
log.info('Bundle {} expected scale {} is met'.format(bundle_id, expected_scale))
return
else:
if bundle_scale > last_scale:
last_scale = bundle_scale
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
last_log_message = 'Bundle {} has scale {}, expected {}'.format(bundle_id, bundle_scale, expected_scale)
log.progress(last_log_message, flush=False)
else:
last_log_message = '{}.'.format(last_log_message)
log.progress(last_log_message, flush=False)
raise WaitTimeoutError('Bundle {} waiting to reach expected scale {}'.format(bundle_id, expected_scale))
def wait_for_condition(bundle_id, condition, condition_name, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
installed_bundles = count_installations(bundle_id, args)
last_log_message = None
if condition(installed_bundles):
log.info('Bundle {} is {}'.format(bundle_id, condition_name))
return
else:
sse_heartbeat_count_after_event = 0
log.info('Bundle {} waiting to be {}'.format(bundle_id, condition_name))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(args.dcos_mode, conduct_url.conductr_host(args), bundle_events_url,
auth=args.conductr_auth, verify=args.server_verification_file)
for event in sse_events:
sse_heartbeat_count_after_event += 1
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError('Bundle {} waiting to be {}'.format(bundle_id, condition_name))
# Check for installed bundles every 3 heartbeats from the last received event.
if event.event or (sse_heartbeat_count_after_event % 3 == 0):
if event.event:
sse_heartbeat_count_after_event = 0
installed_bundles = count_installations(bundle_id, args)
if condition(installed_bundles):
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
log.info('Bundle {} {}'.format(bundle_id, condition_name))
return
else:
if last_log_message:
last_log_message = '{}.'.format(last_log_message)
else:
last_log_message = 'Bundle {} still waiting to be {}'.format(bundle_id, condition_name)
log.progress(last_log_message, flush=False)
raise WaitTimeoutError('Bundle {} waiting to be {}'.format(bundle_id, condition_name))
def test_sse(self):
raw_sse = strip_margin("""|data:
|
|event:My Test Event
|data:My Test Data
|
|data:
|
|""")
raw_sse_iterators = [iter(list(raw_sse))]
iter_content_mock = MagicMock(side_effect=raw_sse_iterators)
raise_for_status_mock = MagicMock()
response_mock = MagicMock()
response_mock.raise_for_status = raise_for_status_mock
response_mock.iter_content = iter_content_mock
request_get_mock = MagicMock(return_value=response_mock)
result = []
with patch('requests.get', request_get_mock):
events = sse_client.get_events(False, '127.0.0.1',
'http://host.com')
for event in events:
result.append(event)
self.assertEqual([
sse_client.Event(event=None, data=''),
sse_client.Event(event='My Test Event', data='My Test Data'),
sse_client.Event(event=None, data='')
], result)
request_get_mock.assert_called_with('http://host.com',
stream=True,
**sse_client.SSE_REQUEST_INPUT)
raise_for_status_mock.assert_called_with()
iter_content_mock.assert_called_with(decode_unicode=True)
def wait_for_scale(bundle_id, expected_scale, wait_for_is_active, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
bundle_scale = get_scale(bundle_id, wait_for_is_active, args)
if bundle_scale == expected_scale:
log.info('Bundle {} expected scale {} is met'.format(
bundle_id, expected_scale))
return
else:
sse_heartbeat_count_after_event = 0
log.info('Bundle {} waiting to reach expected scale {}'.format(
bundle_id, expected_scale))
bundle_events_url = conduct_url.url('bundles/events', args)
sse_events = sse_client.get_events(
args.dcos_mode,
conduct_url.conductr_host(args),
bundle_events_url,
auth=args.conductr_auth,
verify=args.server_verification_file)
last_scale = -1
last_log_message = None
for event in sse_events:
sse_heartbeat_count_after_event += 1
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError(
'Bundle {} waiting to reach expected scale {}'.format(
bundle_id, expected_scale))
# Check for bundle scale every 3 heartbeats from the last received event.
if event.event or (sse_heartbeat_count_after_event % 3 == 0):
if event.event:
sse_heartbeat_count_after_event = 0
bundle_scale = get_scale(bundle_id, wait_for_is_active, args)
if bundle_scale == expected_scale:
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
log.info('Bundle {} expected scale {} is met'.format(
bundle_id, expected_scale))
return
else:
if bundle_scale > last_scale:
last_scale = bundle_scale
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
last_log_message = 'Bundle {} has scale {}, expected {}'.format(
bundle_id, bundle_scale, expected_scale)
log.progress(last_log_message, flush=False)
else:
last_log_message = '{}.'.format(last_log_message)
log.progress(last_log_message, flush=False)
raise WaitTimeoutError(
'Bundle {} waiting to reach expected scale {}'.format(
bundle_id, expected_scale))
def wait_for_deployment_complete(deployment_id, resolved_version, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
def display_bundle_id(bundle_id):
return bundle_id if args.long_ids else bundle_utils.short_id(bundle_id)
def is_completed_with_success(deployment_events):
for event in deployment_events:
if event['eventType'] == 'deploymentSuccess':
return True
return False
def is_completed_with_failure(deployment_events):
for event in deployment_events:
if event['eventType'] == 'deploymentFailure':
return True
return False
def display_deployment_event(deployment_event):
event_type = deployment_event['eventType']
if event_type == 'deploymentStarted':
return 'Deployment started'
elif event_type == 'bundleDownload':
return 'Downloading bundle'
elif event_type == 'configDownload':
compatible_bundle_id = display_bundle_id(
deployment_event['compatibleBundleId'])
return 'Downloading config from bundle {}'.format(
compatible_bundle_id)
elif event_type == 'load':
return 'Loading bundle with config' if 'configFileName' in deployment_event else 'Loading bundle'
elif event_type == 'deploy':
bundle_old = deployment_event['bundleOld']
bundle_new = deployment_event['bundleNew']
deploy_progress = 'Deploying - {} old instance vs {} new instance'.format(
bundle_old['scale'], bundle_new['scale'])
return deploy_progress
elif event_type == 'deploymentSuccess':
return 'Success'
elif event_type == 'deploymentFailure':
return 'Failure: {}'.format(deployment_event['failure'])
else:
return 'Unknown deployment state {}'.format(deployment_event)
def get_event_sequence(deployment_event):
return deployment_event['deploymentSequence']
def log_message(deployment_events):
latest_event = sorted(deployment_events, key=get_event_sequence)[-1]
return display_deployment_event(latest_event)
package_name = resolved_version['package_name']
tag = resolved_version['tag']
bundle_id = display_bundle_id(resolved_version['digest'])
bundle_shorthand = '{}:{}-{}'.format(package_name, tag, bundle_id)
log.info('Deploying {}'.format(bundle_shorthand))
deployment_events = get_deployment_events(deployment_id, args)
if deployment_events and is_completed_with_success(deployment_events):
log.info(log_message(deployment_events))
return
elif deployment_events and is_completed_with_failure(deployment_events):
raise ContinuousDeliveryError('Unable to deploy {} - {}'.format(
bundle_shorthand, log_message(deployment_events)))
else:
sse_heartbeat_count_after_event = 0
deployment_events_url = conduct_url.url('deployments/events', args)
sse_events = sse_client.get_events(
args.dcos_mode,
conduct_url.conductr_host(args),
deployment_events_url,
auth=args.conductr_auth,
verify=args.server_verification_file)
last_events = None
last_log_message = None
for event in sse_events:
sse_heartbeat_count_after_event += 1
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError(
'Deployment is still waiting to be completed')
# Check for deployment state every 3 heartbeats from the last received event.
if event.event or (sse_heartbeat_count_after_event % 3 == 0):
if event.event:
sse_heartbeat_count_after_event = 0
deployment_events = get_deployment_events(deployment_id, args)
if is_completed_with_success(deployment_events):
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
log.info(log_message(deployment_events))
return
elif is_completed_with_failure(deployment_events):
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
raise ContinuousDeliveryError(
'Unable to deploy {} - {}'.format(
bundle_shorthand, log_message(deployment_events)))
else:
if deployment_events != last_events:
last_events = deployment_events
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
last_log_message = log_message(deployment_events)
log.progress(last_log_message, flush=False)
else:
last_log_message = '{}.'.format(last_log_message)
log.progress(last_log_message, flush=False)
raise WaitTimeoutError(
'Deployment for {} is still waiting to be completed')
def wait_for_deployment_complete(deployment_id, resolved_version, args):
log = logging.getLogger(__name__)
start_time = datetime.now()
def display_bundle_id(bundle_id):
return bundle_id if args.long_ids else bundle_utils.short_id(bundle_id)
def is_completed_with_success(deployment_state):
return deployment_state['eventType'] == "deploymentSuccess"
def is_completed_with_failure(deployment_state):
return deployment_state['eventType'] == "deploymentFailure"
def log_message(deployment_state):
event_type = deployment_state['eventType']
if event_type == 'deploymentStarted':
return 'Deployment started'
elif event_type == 'bundleDownload':
return 'Downloading bundle'
elif event_type == 'configDownload':
compatible_bundle_id = display_bundle_id(deployment_state['compatibleBundleId'])
return 'Downloading config from bundle {}'.format(compatible_bundle_id)
elif event_type == 'load':
return 'Loading bundle with config' if 'configFileName' in deployment_state else 'Loading bundle'
elif event_type == 'deploy':
bundle_old = deployment_state['bundleOld']
bundle_new = deployment_state['bundleNew']
deploy_progress = 'Deploying - {} old instance vs {} new instance'.format(bundle_old['scale'],
bundle_new['scale'])
return deploy_progress
elif event_type == 'deploymentSuccess':
return 'Success'
elif event_type == 'deploymentFailure':
return 'Failure: {}'.format(deployment_state['failure'])
else:
return 'Unknown deployment state {}'.format(deployment_state)
package_name = resolved_version['package_name']
compatibility_version = resolved_version['compatibility_version']
bundle_id = display_bundle_id(resolved_version['digest'])
bundle_shorthand = '{}:{}-{}'.format(package_name, compatibility_version, bundle_id)
log.info('Deploying {}'.format(bundle_shorthand))
deployment_state = get_deployment_state(deployment_id, args)
if deployment_state and is_completed_with_success(deployment_state):
log.info(log_message(deployment_state))
return
elif deployment_state and is_completed_with_failure(deployment_state):
raise ContinuousDeliveryError('Unable to deploy {} - {}'.format(bundle_shorthand,
log_message(deployment_state)))
else:
sse_heartbeat_count_after_event = 0
deployment_events_url = conduct_url.url('deployments/events', args)
sse_events = sse_client.get_events(args.dcos_mode, conduct_url.conductr_host(args), deployment_events_url,
auth=args.conductr_auth, verify=args.server_verification_file)
last_deployment_state = None
last_log_message = None
for event in sse_events:
sse_heartbeat_count_after_event += 1
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > args.wait_timeout:
raise WaitTimeoutError('Deployment is still waiting to be completed')
# Check for deployment state every 3 heartbeats from the last received event.
if event.event or (sse_heartbeat_count_after_event % 3 == 0):
if event.event:
sse_heartbeat_count_after_event = 0
deployment_state = get_deployment_state(deployment_id, args)
if is_completed_with_success(deployment_state):
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
log.info(log_message(deployment_state))
return
elif is_completed_with_failure(deployment_state):
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
raise ContinuousDeliveryError('Unable to deploy {} - {}'.format(bundle_shorthand,
log_message(deployment_state)))
else:
if deployment_state != last_deployment_state:
last_deployment_state = deployment_state
# Reprint previous message with flush to go to next line
if last_log_message:
log.progress(last_log_message, flush=True)
last_log_message = log_message(deployment_state)
log.progress(last_log_message, flush=False)
else:
last_log_message = '{}.'.format(last_log_message)
log.progress(last_log_message, flush=False)
raise WaitTimeoutError('Deployment for {} is still waiting to be completed')
