def test_get_cluster_instance_map_for_service(): fake_service_configs = [ MarathonServiceConfig( service='service1', cluster='clusterA', instance='main', branch_dict={}, config_dict={'deploy_group': 'no_thanks'}, ), MarathonServiceConfig( service='service1', cluster='clusterB', instance='main', branch_dict={}, config_dict={'deploy_group': 'try_me'}, ), MarathonServiceConfig( service='service1', cluster='clusterA', instance='canary', branch_dict={}, config_dict={'deploy_group': 'try_me'}, ), ] with mock.patch( 'paasta_tools.generate_deployments_for_service.get_instance_configs_for_service', return_value=fake_service_configs, autospec=True, ): ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1', 'try_me') expected = {'clusterA': {'instances': ['canary']}, 'clusterB': {'instances': ['main']}} assert ret == expected ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1') expected = {'clusterA': {'instances': ['main', 'canary']}, 'clusterB': {'instances': ['main']}} assert ret == expected
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout): cluster_map = get_cluster_instance_map_for_service( soa_dir=soa_dir, service=service, deploy_group=deploy_group) if not cluster_map: _log(service=service, component='deploy', line=("Couldn't find any instances for service {0} in deploy " "group {1}".format(service, deploy_group)), level='event') raise NoInstancesFound paasta_print("Waiting for deployment of {0} for '{1}' complete...".format( git_sha, deploy_group)) total_instances = 0 clusters_data = [] for cluster in cluster_map: clusters_data.append( ClusterData(cluster=cluster, service=service, git_sha=git_sha, instances_queue=Queue())) for i in cluster_map[cluster]['instances']: clusters_data[-1].instances_queue.put(i) total_instances += len(cluster_map[cluster]['instances']) deadline = time.time() + timeout green_light = Event() green_light.set() with progressbar.ProgressBar(maxval=total_instances) as bar: while time.time() < deadline: _query_clusters(clusters_data, green_light) if not green_light.is_set(): raise KeyboardInterrupt bar.update(total_instances - sum((c.instances_queue.qsize() for c in clusters_data))) if all((cluster.instances_queue.empty() for cluster in clusters_data)): sys.stdout.flush() return 0 else: time.sleep(min(10, timeout)) sys.stdout.flush() _log(service=service, component='deploy', line=( "\n\nTimed out after {0} seconds, waiting for {2} in {1} to be " "deployed by PaaSTA. \n\n" "This probably means the deploy hasn't suceeded. The new service " "might not be healthy or one or more clusters could be having " "issues.\n\n" "To debug: try running:\n\n paasta status -s {2} -vv\n" " paasta logs -s {2}\n\nto determine the cause.\n\n" "If the service is known to be slow to start you may wish to " "increase the timeout on this step.".format( timeout, deploy_group, service)), level='event') raise TimeoutError
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout): cluster_map = get_cluster_instance_map_for_service(soa_dir, service, deploy_group) if not cluster_map: line = "Couldn't find any instances for service {0} in deploy group {1}".format( service, deploy_group) _log(service=service, component='deploy', line=line, level='event') raise NoInstancesFound for cluster in cluster_map.values(): cluster['deployed'] = 0 try: with Timeout(seconds=timeout): total_instances = sum( [len(v["instances"]) for v in cluster_map.values()]) with progressbar.ProgressBar(maxval=total_instances) as bar: while True: for cluster, instances in cluster_map.items(): if cluster_map[cluster]['deployed'] != len( cluster_map[cluster]['instances']): cluster_map[cluster][ 'deployed'] = instances_deployed( cluster=cluster, service=service, instances=instances['instances'], git_sha=git_sha) if cluster_map[cluster]['deployed'] == len( cluster_map[cluster]['instances']): instance_csv = ", ".join( cluster_map[cluster]['instances']) print "Deploy to %s complete! (instances: %s)" % ( cluster, instance_csv) bar.update( sum([v["deployed"] for v in cluster_map.values()])) if all([ cluster['deployed'] == len(cluster["instances"]) for cluster in cluster_map.values() ]): break else: time.sleep(10) except TimeoutError: human_status = [ "{0}: {1}".format(cluster, data['deployed']) for cluster, data in cluster_map.items() ] line = "\nCurrent deployment status of {0} per cluster:\n".format( deploy_group) + "\n".join(human_status) _log(service=service, component='deploy', line=line, level='event') line = "\n\nTimed out after {0} seconds, waiting for {1} in {2} to be deployed by PaaSTA. \n\n"\ "This probably means the deploy hasn't suceeded. The new service might not be healthy or one "\ "or more clusters could be having issues.\n\n"\ "To debug: try running 'paasta status -s {2} -vv' or 'paasta logs -s {2}' to determine the cause.\n\n"\ "{3} is still *marked* for deployment. To rollback, you can run: 'paasta rollback --service "\ "{2} --deploy-group {1}'\n\n"\ "If the service is known to be slow to start you may wish to increase "\ "the timeout on this step.".format(timeout, deploy_group, service, git_sha) _log(service=service, component='deploy', line=line, level='event') raise return True
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout): cluster_map = get_cluster_instance_map_for_service(soa_dir, service, deploy_group) if not cluster_map: line = "Couldn't find any instances for service {0} in deploy group {1}".format(service, deploy_group) _log( service=service, component='deploy', line=line, level='event' ) raise NoInstancesFound for cluster in cluster_map.values(): cluster['deployed'] = 0 try: with Timeout(seconds=timeout): total_instances = sum([len(v["instances"]) for v in cluster_map.values()]) with progressbar.ProgressBar(maxval=total_instances) as bar: while True: for cluster, instances in cluster_map.items(): if cluster_map[cluster]['deployed'] != len(cluster_map[cluster]['instances']): cluster_map[cluster]['deployed'] = instances_deployed( cluster=cluster, service=service, instances=instances['instances'], git_sha=git_sha) if cluster_map[cluster]['deployed'] == len(cluster_map[cluster]['instances']): instance_csv = ", ".join(cluster_map[cluster]['instances']) print "Deploy to %s complete! (instances: %s)" % (cluster, instance_csv) bar.update(sum([v["deployed"] for v in cluster_map.values()])) if all([cluster['deployed'] == len(cluster["instances"]) for cluster in cluster_map.values()]): break else: time.sleep(10) except TimeoutError: human_status = ["{0}: {1}".format(cluster, data['deployed']) for cluster, data in cluster_map.items()] line = "\nCurrent deployment status of {0} per cluster:\n".format(deploy_group) + "\n".join(human_status) _log( service=service, component='deploy', line=line, level='event' ) line = "\n\nTimed out after {0} seconds, waiting for {1} in {2} to be deployed by PaaSTA. \n\n"\ "This probably means the deploy hasn't suceeded. The new service might not be healthy or one "\ "or more clusters could be having issues.\n\n"\ "To debug: try running 'paasta status -s {2} -vv' or 'paasta logs -s {2}' to determine the cause.\n\n"\ "{3} is still *marked* for deployment. To rollback, you can run: 'paasta rollback --service "\ "{2} --deploy-group {1}'\n\n"\ "If the service is known to be slow to start you may wish to increase "\ "the timeout on this step.".format(timeout, deploy_group, service, git_sha) _log( service=service, component='deploy', line=line, level='event' ) raise return True
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout): cluster_map = get_cluster_instance_map_for_service(soa_dir, service, deploy_group) if not cluster_map: line = "Couldn't find any instances for service {0} in deploy group {1}".format(service, deploy_group) _log( service=service, component='deploy', line=line, level='event' ) raise NoInstancesFound paasta_print("Waiting for deployment of {0} for '{1}' complete..." .format(git_sha, deploy_group)) for cluster in cluster_map.values(): cluster['deployed'] = 0 try: with Timeout(seconds=timeout): total_instances = sum([len(v["instances"]) for v in cluster_map.values()]) with progressbar.ProgressBar(maxval=total_instances) as bar: while True: for cluster, instances in cluster_map.items(): if cluster_map[cluster]['deployed'] != len(cluster_map[cluster]['instances']): cluster_map[cluster]['deployed'] = instances_deployed( cluster=cluster, service=service, instances=instances['instances'], git_sha=git_sha) if cluster_map[cluster]['deployed'] == len(cluster_map[cluster]['instances']): instance_csv = ", ".join(cluster_map[cluster]['instances']) paasta_print("Deploy to %s complete! (instances: %s)" % (cluster, instance_csv)) bar.update(sum([v["deployed"] for v in cluster_map.values()])) if all([cluster['deployed'] == len(cluster["instances"]) for cluster in cluster_map.values()]): sys.stdout.flush() break else: time.sleep(10) sys.stdout.flush() except TimeoutError: line = "\n\nTimed out after {0} seconds, waiting for {2} in {1} to be deployed by PaaSTA. \n\n"\ "This probably means the deploy hasn't suceeded. The new service might not be healthy or one "\ "or more clusters could be having issues.\n\n"\ "To debug: try running:\n\n"\ " paasta status -s {2} -vv\n"\ " paasta logs -s {2}\n\n"\ "to determine the cause.\n\n"\ "If the service is known to be slow to start you may wish to increase "\ "the timeout on this step.".format(timeout, deploy_group, service) _log( service=service, component='deploy', line=line, level='event' ) raise return True
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout): cluster_map = get_cluster_instance_map_for_service( soa_dir=soa_dir, service=service, deploy_group=deploy_group) if not cluster_map: _log(service=service, component='deploy', line=("Couldn't find any instances for service {} in deploy " "group {}".format(service, deploy_group)), level='event') raise NoInstancesFound paasta_print("Waiting for deployment of {} for '{}' complete...".format( git_sha, deploy_group)) total_instances = 0 clusters_data = [] for cluster in cluster_map: clusters_data.append( ClusterData(cluster=cluster, service=service, git_sha=git_sha, instances_queue=Queue())) for i in cluster_map[cluster]['instances']: clusters_data[-1].instances_queue.put(i) total_instances += len(cluster_map[cluster]['instances']) deadline = time.time() + timeout green_light = Event() green_light.set() with progressbar.ProgressBar(maxval=total_instances) as bar: while time.time() < deadline: _query_clusters(clusters_data, green_light) if not green_light.is_set(): raise KeyboardInterrupt bar.update(total_instances - sum((c.instances_queue.qsize() for c in clusters_data))) if all((cluster.instances_queue.empty() for cluster in clusters_data)): sys.stdout.flush() return 0 else: time.sleep(min(60, timeout)) sys.stdout.flush() _log(service=service, component='deploy', line=compose_timeout_message(clusters_data, timeout, deploy_group, service, git_sha), level='event') raise TimeoutError
def test_get_cluster_instance_map_for_service(): fake_service_configs = [ MarathonServiceConfig( service='service1', cluster='clusterA', instance='main', branch_dict={}, config_dict={'deploy_group': 'no_thanks'}, ), MarathonServiceConfig( service='service1', cluster='clusterB', instance='main', branch_dict={}, config_dict={'deploy_group': 'try_me'}, ), MarathonServiceConfig( service='service1', cluster='clusterA', instance='canary', branch_dict={}, config_dict={'deploy_group': 'try_me'}, ), ] with contextlib.nested( mock.patch('paasta_tools.generate_deployments_for_service.get_instance_configs_for_service', return_value=fake_service_configs, autospec=True), ) as ( mock_get_instance_configs_for_service, ): ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1', 'try_me') mock_get_instance_configs_for_service.assert_called_with(soa_dir='/nail/blah', service='service1') expected = {'clusterA': {'instances': ['canary']}, 'clusterB': {'instances': ['main']}} assert ret == expected ret = generate_deployments_for_service.get_cluster_instance_map_for_service('/nail/blah', 'service1') expected = {'clusterA': {'instances': ['main', 'canary']}, 'clusterB': {'instances': ['main']}} assert ret == expected
def paasta_args_mixer(args, service): if args.deploy_group is None: cluster_whitelist = args.clusters.split( ",") if args.clusters is not None else [] if args.instances is not None: instance_whitelist = verify_instances(args.instances, service, cluster_whitelist) else: instance_whitelist = [] else: clusters_instances = get_cluster_instance_map_for_service( soa_dir=args.soa_dir, service=service, deploy_group=args.deploy_group) cluster_whitelist = list(( set(args.clusters.split(",")) & set(clusters_instances) ) if args.clusters is not None else clusters_instances) diff = set(cluster_whitelist) - set(clusters_instances) if (args.clusters is not None and not cluster_whitelist) or diff: paasta_print( "The %s deploy_group doesn't have any instances of %s on %s." % (args.deploy_group, service, ', '.join(diff) or args.clusters)) return None instances_set = set() for cluster in cluster_whitelist: for instance in clusters_instances[cluster].get('instances', []): instances_set.add(instance) instance_whitelist = list( set(args.instances.split(",")) & instances_set if args.instances is not None else instances_set) diff = set(instance_whitelist) - instances_set if (args.instances is not None and not instance_whitelist) or diff: paasta_print( "The %s deploy_group doesn't have any instances of %s matching %s." % (args.deploy_group, service, ', '.join(diff) or args.instances)) return None PaastaArgs = namedtuple('PaastaArgs', ['cluster_whitelist', 'instance_whitelist']) return PaastaArgs(cluster_whitelist=cluster_whitelist, instance_whitelist=instance_whitelist)
def wait_for_deployment(service, deploy_group, git_sha, soa_dir, timeout): # Currently only 'marathon' instances are supported for wait_for_deployment because they # are the only thing that are worth waiting on. cluster_map = get_cluster_instance_map_for_service( soa_dir=soa_dir, service=service, deploy_group=deploy_group, type_filter='marathon', ) if not cluster_map: _log( service=service, component='deploy', line=("Couldn't find any marathon instances for service {} in deploy group {}. Exiting.".format( service, deploy_group, )), level='event', ) return paasta_print("Waiting for deployment of {} for '{}' to complete..." .format(git_sha, deploy_group)) total_instances = 0 clusters_data = [] api_endpoints = load_system_paasta_config().get_api_endpoints() for cluster in cluster_map: if cluster not in api_endpoints: paasta_print(PaastaColors.red( 'Cluster %s is NOT in paasta-api endpoints config.' % cluster, )) raise NoSuchCluster clusters_data.append(ClusterData( cluster=cluster, service=service, git_sha=git_sha, instances_queue=Queue(), soa_dir=soa_dir, )) for i in cluster_map[cluster]['instances']: clusters_data[-1].instances_queue.put(i) total_instances += len(cluster_map[cluster]['instances']) deadline = time.time() + timeout green_light = Event() green_light.set() with progressbar.ProgressBar(maxval=total_instances) as bar: while time.time() < deadline: _query_clusters(clusters_data, green_light) if not green_light.is_set(): raise KeyboardInterrupt bar.update(total_instances - sum(( c.instances_queue.qsize() for c in clusters_data ))) if all(( cluster.instances_queue.empty() for cluster in clusters_data )): sys.stdout.flush() return 0 else: time.sleep(min(60, timeout)) sys.stdout.flush() _log( service=service, component='deploy', line=compose_timeout_message(clusters_data, timeout, deploy_group, service, git_sha), level='event', ) raise TimeoutError