def test_get_cluster_instance_map_for_service():
fake_service_configs = [
MarathonServiceConfig(
service='service1',
cluster='clusterA',
instance='main',
branch_dict={},
config_dict={'deploy_group': 'no_thanks'},
),
MarathonServiceConfig(
service='service1',
cluster='clusterB',
instance='main',
branch_dict={},
config_dict={'deploy_group': 'try_me'},
),
MarathonServiceConfig(
service='service1',
cluster='clusterA',
instance='canary',
branch_dict={},
config_dict={'deploy_group': 'try_me'},
),
]
with mock.patch(
'paasta_tools.generate_deployments_for_service.get_instance_configs_for_service',
return_value=fake_service_configs, autospec=True,
):
ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1', 'try_me')
expected = {'clusterA': {'instances': ['canary']}, 'clusterB': {'instances': ['main']}}
assert ret == expected
ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1')
expected = {'clusterA': {'instances': ['main', 'canary']}, 'clusterB': {'instances': ['main']}}
assert ret == expected
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout):
cluster_map = get_cluster_instance_map_for_service(
soa_dir=soa_dir, service=service, deploy_group=deploy_group)
if not cluster_map:
_log(service=service,
component='deploy',
line=("Couldn't find any instances for service {0} in deploy "
"group {1}".format(service, deploy_group)),
level='event')
raise NoInstancesFound
paasta_print("Waiting for deployment of {0} for '{1}' complete...".format(
git_sha, deploy_group))
total_instances = 0
clusters_data = []
for cluster in cluster_map:
clusters_data.append(
ClusterData(cluster=cluster,
service=service,
git_sha=git_sha,
instances_queue=Queue()))
for i in cluster_map[cluster]['instances']:
clusters_data[-1].instances_queue.put(i)
total_instances += len(cluster_map[cluster]['instances'])
deadline = time.time() + timeout
green_light = Event()
green_light.set()
with progressbar.ProgressBar(maxval=total_instances) as bar:
while time.time() < deadline:
_query_clusters(clusters_data, green_light)
if not green_light.is_set():
raise KeyboardInterrupt
bar.update(total_instances - sum((c.instances_queue.qsize()
for c in clusters_data)))
if all((cluster.instances_queue.empty()
for cluster in clusters_data)):
sys.stdout.flush()
return 0
else:
time.sleep(min(10, timeout))
sys.stdout.flush()
_log(service=service,
component='deploy',
line=(
"\n\nTimed out after {0} seconds, waiting for {2} in {1} to be "
"deployed by PaaSTA. \n\n"
"This probably means the deploy hasn't suceeded. The new service "
"might not be healthy or one or more clusters could be having "
"issues.\n\n"
"To debug: try running:\n\n paasta status -s {2} -vv\n"
" paasta logs -s {2}\n\nto determine the cause.\n\n"
"If the service is known to be slow to start you may wish to "
"increase the timeout on this step.".format(
timeout, deploy_group, service)),
level='event')
raise TimeoutError
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout):
cluster_map = get_cluster_instance_map_for_service(soa_dir, service,
deploy_group)
if not cluster_map:
line = "Couldn't find any instances for service {0} in deploy group {1}".format(
service, deploy_group)
_log(service=service, component='deploy', line=line, level='event')
raise NoInstancesFound
for cluster in cluster_map.values():
cluster['deployed'] = 0
try:
with Timeout(seconds=timeout):
total_instances = sum(
[len(v["instances"]) for v in cluster_map.values()])
with progressbar.ProgressBar(maxval=total_instances) as bar:
while True:
for cluster, instances in cluster_map.items():
if cluster_map[cluster]['deployed'] != len(
cluster_map[cluster]['instances']):
cluster_map[cluster][
'deployed'] = instances_deployed(
cluster=cluster,
service=service,
instances=instances['instances'],
git_sha=git_sha)
if cluster_map[cluster]['deployed'] == len(
cluster_map[cluster]['instances']):
instance_csv = ", ".join(
cluster_map[cluster]['instances'])
print "Deploy to %s complete! (instances: %s)" % (
cluster, instance_csv)
bar.update(
sum([v["deployed"] for v in cluster_map.values()]))
if all([
cluster['deployed'] == len(cluster["instances"])
for cluster in cluster_map.values()
]):
break
else:
time.sleep(10)
except TimeoutError:
human_status = [
"{0}: {1}".format(cluster, data['deployed'])
for cluster, data in cluster_map.items()
]
line = "\nCurrent deployment status of {0} per cluster:\n".format(
deploy_group) + "\n".join(human_status)
_log(service=service, component='deploy', line=line, level='event')
line = "\n\nTimed out after {0} seconds, waiting for {1} in {2} to be deployed by PaaSTA. \n\n"\
"This probably means the deploy hasn't suceeded. The new service might not be healthy or one "\
"or more clusters could be having issues.\n\n"\
"To debug: try running 'paasta status -s {2} -vv' or 'paasta logs -s {2}' to determine the cause.\n\n"\
"{3} is still *marked* for deployment. To rollback, you can run: 'paasta rollback --service "\
"{2} --deploy-group {1}'\n\n"\
"If the service is known to be slow to start you may wish to increase "\
"the timeout on this step.".format(timeout, deploy_group, service, git_sha)
_log(service=service, component='deploy', line=line, level='event')
raise
return True
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout):
cluster_map = get_cluster_instance_map_for_service(soa_dir, service, deploy_group)
if not cluster_map:
line = "Couldn't find any instances for service {0} in deploy group {1}".format(service, deploy_group)
_log(
service=service,
component='deploy',
line=line,
level='event'
)
raise NoInstancesFound
for cluster in cluster_map.values():
cluster['deployed'] = 0
try:
with Timeout(seconds=timeout):
total_instances = sum([len(v["instances"]) for v in cluster_map.values()])
with progressbar.ProgressBar(maxval=total_instances) as bar:
while True:
for cluster, instances in cluster_map.items():
if cluster_map[cluster]['deployed'] != len(cluster_map[cluster]['instances']):
cluster_map[cluster]['deployed'] = instances_deployed(
cluster=cluster,
service=service,
instances=instances['instances'],
git_sha=git_sha)
if cluster_map[cluster]['deployed'] == len(cluster_map[cluster]['instances']):
instance_csv = ", ".join(cluster_map[cluster]['instances'])
print "Deploy to %s complete! (instances: %s)" % (cluster, instance_csv)
bar.update(sum([v["deployed"] for v in cluster_map.values()]))
if all([cluster['deployed'] == len(cluster["instances"]) for cluster in cluster_map.values()]):
break
else:
time.sleep(10)
except TimeoutError:
human_status = ["{0}: {1}".format(cluster, data['deployed']) for cluster, data in cluster_map.items()]
line = "\nCurrent deployment status of {0} per cluster:\n".format(deploy_group) + "\n".join(human_status)
_log(
service=service,
component='deploy',
line=line,
level='event'
)
line = "\n\nTimed out after {0} seconds, waiting for {1} in {2} to be deployed by PaaSTA. \n\n"\
"This probably means the deploy hasn't suceeded. The new service might not be healthy or one "\
"or more clusters could be having issues.\n\n"\
"To debug: try running 'paasta status -s {2} -vv' or 'paasta logs -s {2}' to determine the cause.\n\n"\
"{3} is still *marked* for deployment. To rollback, you can run: 'paasta rollback --service "\
"{2} --deploy-group {1}'\n\n"\
"If the service is known to be slow to start you may wish to increase "\
"the timeout on this step.".format(timeout, deploy_group, service, git_sha)
_log(
service=service,
component='deploy',
line=line,
level='event'
)
raise
return True
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout):
cluster_map = get_cluster_instance_map_for_service(soa_dir, service, deploy_group)
if not cluster_map:
line = "Couldn't find any instances for service {0} in deploy group {1}".format(service, deploy_group)
_log(
service=service,
component='deploy',
line=line,
level='event'
)
raise NoInstancesFound
paasta_print("Waiting for deployment of {0} for '{1}' complete..."
.format(git_sha, deploy_group))
for cluster in cluster_map.values():
cluster['deployed'] = 0
try:
with Timeout(seconds=timeout):
total_instances = sum([len(v["instances"]) for v in cluster_map.values()])
with progressbar.ProgressBar(maxval=total_instances) as bar:
while True:
for cluster, instances in cluster_map.items():
if cluster_map[cluster]['deployed'] != len(cluster_map[cluster]['instances']):
cluster_map[cluster]['deployed'] = instances_deployed(
cluster=cluster,
service=service,
instances=instances['instances'],
git_sha=git_sha)
if cluster_map[cluster]['deployed'] == len(cluster_map[cluster]['instances']):
instance_csv = ", ".join(cluster_map[cluster]['instances'])
paasta_print("Deploy to %s complete! (instances: %s)" % (cluster, instance_csv))
bar.update(sum([v["deployed"] for v in cluster_map.values()]))
if all([cluster['deployed'] == len(cluster["instances"]) for cluster in cluster_map.values()]):
sys.stdout.flush()
break
else:
time.sleep(10)
sys.stdout.flush()
except TimeoutError:
line = "\n\nTimed out after {0} seconds, waiting for {2} in {1} to be deployed by PaaSTA. \n\n"\
"This probably means the deploy hasn't suceeded. The new service might not be healthy or one "\
"or more clusters could be having issues.\n\n"\
"To debug: try running:\n\n"\
" paasta status -s {2} -vv\n"\
" paasta logs -s {2}\n\n"\
"to determine the cause.\n\n"\
"If the service is known to be slow to start you may wish to increase "\
"the timeout on this step.".format(timeout, deploy_group, service)
_log(
service=service,
component='deploy',
line=line,
level='event'
)
raise
return True
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout):
cluster_map = get_cluster_instance_map_for_service(
soa_dir=soa_dir, service=service, deploy_group=deploy_group)
if not cluster_map:
_log(service=service,
component='deploy',
line=("Couldn't find any instances for service {} in deploy "
"group {}".format(service, deploy_group)),
level='event')
raise NoInstancesFound
paasta_print("Waiting for deployment of {} for '{}' complete...".format(
git_sha, deploy_group))
total_instances = 0
clusters_data = []
for cluster in cluster_map:
clusters_data.append(
ClusterData(cluster=cluster,
service=service,
git_sha=git_sha,
instances_queue=Queue()))
for i in cluster_map[cluster]['instances']:
clusters_data[-1].instances_queue.put(i)
total_instances += len(cluster_map[cluster]['instances'])
deadline = time.time() + timeout
green_light = Event()
green_light.set()
with progressbar.ProgressBar(maxval=total_instances) as bar:
while time.time() < deadline:
_query_clusters(clusters_data, green_light)
if not green_light.is_set():
raise KeyboardInterrupt
bar.update(total_instances - sum((c.instances_queue.qsize()
for c in clusters_data)))
if all((cluster.instances_queue.empty()
for cluster in clusters_data)):
sys.stdout.flush()
return 0
else:
time.sleep(min(60, timeout))
sys.stdout.flush()
_log(service=service,
component='deploy',
line=compose_timeout_message(clusters_data, timeout, deploy_group,
service, git_sha),
level='event')
raise TimeoutError
def test_get_cluster_instance_map_for_service():
fake_service_configs = [
MarathonServiceConfig(
service='service1',
cluster='clusterA',
instance='main',
branch_dict={},
config_dict={'deploy_group': 'no_thanks'},
),
MarathonServiceConfig(
service='service1',
cluster='clusterB',
instance='main',
branch_dict={},
config_dict={'deploy_group': 'try_me'},
),
MarathonServiceConfig(
service='service1',
cluster='clusterA',
instance='canary',
branch_dict={},
config_dict={'deploy_group': 'try_me'},
),
]
with contextlib.nested(
mock.patch('paasta_tools.generate_deployments_for_service.get_instance_configs_for_service',
return_value=fake_service_configs, autospec=True),
) as (
mock_get_instance_configs_for_service,
):
ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1', 'try_me')
mock_get_instance_configs_for_service.assert_called_with(soa_dir='/nail/blah', service='service1')
expected = {'clusterA': {'instances': ['canary']}, 'clusterB': {'instances': ['main']}}
assert ret == expected
ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1')
expected = {'clusterA': {'instances': ['main', 'canary']}, 'clusterB': {'instances': ['main']}}
assert ret == expected
def paasta_args_mixer(args, service):
if args.deploy_group is None:
cluster_whitelist = args.clusters.split(
",") if args.clusters is not None else []
if args.instances is not None:
instance_whitelist = verify_instances(args.instances, service,
cluster_whitelist)
else:
instance_whitelist = []
else:
clusters_instances = get_cluster_instance_map_for_service(
soa_dir=args.soa_dir,
service=service,
deploy_group=args.deploy_group)
cluster_whitelist = list((
set(args.clusters.split(",")) & set(clusters_instances)
) if args.clusters is not None else clusters_instances)
diff = set(cluster_whitelist) - set(clusters_instances)
if (args.clusters is not None and not cluster_whitelist) or diff:
paasta_print(
"The %s deploy_group doesn't have any instances of %s on %s." %
(args.deploy_group, service, ', '.join(diff) or args.clusters))
return None
instances_set = set()
for cluster in cluster_whitelist:
for instance in clusters_instances[cluster].get('instances', []):
instances_set.add(instance)
instance_whitelist = list(
set(args.instances.split(","))
& instances_set if args.instances is not None else instances_set)
diff = set(instance_whitelist) - instances_set
if (args.instances is not None and not instance_whitelist) or diff:
paasta_print(
"The %s deploy_group doesn't have any instances of %s matching %s."
% (args.deploy_group, service, ', '.join(diff)
or args.instances))
return None
PaastaArgs = namedtuple('PaastaArgs',
['cluster_whitelist', 'instance_whitelist'])
return PaastaArgs(cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist)
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout):
# Currently only 'marathon' instances are supported for wait_for_deployment because they
# are the only thing that are worth waiting on.
cluster_map = get_cluster_instance_map_for_service(
soa_dir=soa_dir, service=service, deploy_group=deploy_group, type_filter='marathon',
)
if not cluster_map:
_log(
service=service,
component='deploy',
line=("Couldn't find any marathon instances for service {} in deploy group {}. Exiting.".format(
service,
deploy_group,
)),
level='event',
)
return
paasta_print("Waiting for deployment of {} for '{}' to complete..."
.format(git_sha, deploy_group))
total_instances = 0
clusters_data = []
api_endpoints = load_system_paasta_config().get_api_endpoints()
for cluster in cluster_map:
if cluster not in api_endpoints:
paasta_print(PaastaColors.red(
'Cluster %s is NOT in paasta-api endpoints config.' %
cluster,
))
raise NoSuchCluster
clusters_data.append(ClusterData(
cluster=cluster, service=service,
git_sha=git_sha,
instances_queue=Queue(),
soa_dir=soa_dir,
))
for i in cluster_map[cluster]['instances']:
clusters_data[-1].instances_queue.put(i)
total_instances += len(cluster_map[cluster]['instances'])
deadline = time.time() + timeout
green_light = Event()
green_light.set()
with progressbar.ProgressBar(maxval=total_instances) as bar:
while time.time() < deadline:
_query_clusters(clusters_data, green_light)
if not green_light.is_set():
raise KeyboardInterrupt
bar.update(total_instances - sum((
c.instances_queue.qsize()
for c in clusters_data
)))
if all((
cluster.instances_queue.empty()
for cluster in clusters_data
)):
sys.stdout.flush()
return 0
else:
time.sleep(min(60, timeout))
sys.stdout.flush()
_log(
service=service,
component='deploy',
line=compose_timeout_message(clusters_data, timeout, deploy_group, service, git_sha),
level='event',
)
raise TimeoutError
