def after_scenario(context, scenario): if getattr(context, "tmpdir", None): shutil.rmtree(context.tmpdir) if getattr(context, "running_container_id", None): docker_client = get_docker_client() docker_client.stop(container=context.running_container_id) docker_client.remove_container(container=context.running_container_id)
async def main(): args = parse_args() docker_client = get_docker_client() running_mesos_task_ids = [task["id"] for task in mesos_tools.filter_running_tasks( await mesos_tools.get_running_tasks_from_frameworks(''), )] running_mesos_docker_containers = get_running_mesos_docker_containers() orphaned_containers = [] for container in running_mesos_docker_containers: mesos_task_id = mesos_tools.get_mesos_id_from_container( container=container, client=docker_client, ) if mesos_task_id not in running_mesos_task_ids: orphaned_containers.append((container["Names"][0].strip("/"), mesos_task_id)) if orphaned_containers: paasta_print("CRIT: Docker containers are orphaned: {}{}".format( ", ".join( f"{container_name} ({mesos_task_id})" for container_name, mesos_task_id in orphaned_containers ), " and will be killed" if args.force else "", )) if args.force: for container_name, mesos_task_id in orphaned_containers: docker_client.kill(container_name) sys.exit(1) else: paasta_print("OK: All mesos task IDs accounted for") sys.exit(0)
def main(): args = parse_args() if not args.mesos_id: sys.stdout.write( "The Mesos task id you supplied seems to be an empty string! Please provide a valid task id.\n") sys.exit(2) docker_client = get_docker_client() container_id = get_container_id_for_mesos_id(docker_client, args.mesos_id) if container_id: try: with time_limit(args.timeout): output, return_code = execute_in_container(docker_client, container_id, args.cmd, args.timeout) sys.stdout.write(output) except TimeoutException: sys.stdout.write("Command timed out!\n") return_code = 1 finally: sys.exit(return_code) else: sys.stdout.write("Could not find container with MESOS_TASK_ID '%s'.\n" % args.mesos_id) sys.exit(1)
def main(): args = parse_args() if not args.mesos_id: paasta_print( "The Mesos task id you supplied seems to be an empty string! Please provide a valid task id." ) sys.exit(2) docker_client = get_docker_client() container_id = get_container_id_for_mesos_id(docker_client, args.mesos_id) if container_id: try: with time_limit(args.timeout): output, return_code = execute_in_container( docker_client, container_id, args.cmd, args.timeout) paasta_print(output) except TimeoutException: paasta_print("Command timed out!") return_code = 1 finally: sys.exit(return_code) else: paasta_print("Could not find container with MESOS_TASK_ID '%s'." % args.mesos_id) sys.exit(1)
def main(): args = parse_args() docker_client = get_docker_client() running_mesos_task_ids = [task["id"] for task in mesos_tools.filter_running_tasks( mesos_tools.get_running_tasks_from_active_frameworks(''))] running_mesos_docker_containers = get_running_mesos_docker_containers() orphaned_containers = [] for container in running_mesos_docker_containers: mesos_task_id = mesos_tools.get_mesos_id_from_container( container=container, client=docker_client) if mesos_task_id not in running_mesos_task_ids: orphaned_containers.append((container["Names"][0].strip("/"), mesos_task_id)) if orphaned_containers: print "CRIT: Docker containers are orphaned: %s%s" % (", ".join( "%s (%s)" % (container_name, mesos_task_id) for container_name, mesos_task_id in orphaned_containers ), " and will be killed" if args.force else "") if args.force: for container_name, mesos_task_id in orphaned_containers: docker_client.kill(container_name) sys.exit(1) else: print "OK: All mesos task IDs accounted for" sys.exit(0)
def main(): docker_client = get_docker_client() ip_to_containers = defaultdict(list) for container in docker_client.containers(): networks = container['NetworkSettings']['Networks'] if 'bridge' in networks: ip = networks['bridge']['IPAddress'] if ip: ip_to_containers[ip].append(container) output = [] for ip, containers in ip_to_containers.items(): if len(containers) > 1: output.append(f'{ip} shared by the following containers:') for container in containers: output.append(' Image: {}'.format(container['Image'])) output.append(' ID: {}'.format(container['Id'])) output.append(' State: {}'.format(container['State'])) output.append(' Status: {}'.format(container['Status'])) output.append('') if output: print( 'CRITICAL - There are multiple Docker containers assigned to the same IP.' ) print( 'There should only be one per IP. Choose one to keep and try stopping the others.' ) print('\n'.join(output)) return 2 else: print('OK - No Docker containers sharing an IP on this host.') return 0
def main(): parser = argparse.ArgumentParser(description=( 'Script to dump a HAProxy map between container IPs and task IDs.'), ) parser.add_argument( 'map_file', nargs='?', default='/var/run/synapse/maps/ip_to_service.map', help='Where to write the output map file', ) args = parser.parse_args() prev_ip_to_task_id = get_prev_file_contents(args.map_file) new_lines = [] ip_addrs = [] service_ips_and_ids = extract_taskid_and_ip(get_docker_client()) for ip_addr, task_id in service_ips_and_ids: ip_addrs.append(ip_addr) update_haproxy_mapping(ip_addr, task_id, prev_ip_to_task_id, args.map_file) new_lines.append(f'{ip_addr} {task_id}') remove_stopped_container_entries(prev_ip_to_task_id.keys(), ip_addrs, args.map_file) # Replace the file contents with the new map with atomic_file_write(args.map_file) as fp: fp.write('\n'.join(new_lines))
def main(): try: from clog.loggers import ScribeLogger except ImportError: print("Scribe logger unavailable, exiting.", file=sys.stderr) sys.exit(1) scribe_logger = ScribeLogger(host="169.254.255.254", port=1463, retry_interval=5) cluster = load_system_paasta_config().get_cluster() client = get_docker_client() for ( timestamp, hostname, container_id, process_name, ) in capture_oom_events_from_stdin(): try: docker_inspect = client.inspect_container(resource_id=container_id) except (APIError): continue env_vars = get_container_env_as_dict(docker_inspect) service = env_vars.get("PAASTA_SERVICE", "unknown") instance = env_vars.get("PAASTA_INSTANCE", "unknown") log_line = LogLine( timestamp=timestamp, hostname=hostname, container_id=container_id, cluster=cluster, service=service, instance=instance, process_name=process_name, ) log_to_scribe(scribe_logger, log_line) log_to_paasta(log_line) send_sfx_event(service, instance, cluster)
def after_scenario(context, scenario): if getattr(context, "tmpdir", None): shutil.rmtree(context.tmpdir) if getattr(context, "running_container_id", None): docker_client = get_docker_client() docker_client.stop(container=context.running_container_id) docker_client.remove_container(container=context.running_container_id) if getattr(context, "fake_http_server", None): context.fake_http_server.shutdown() context.fake_http_server = None
def main(): docker_client = get_docker_client() ip_to_containers = defaultdict(list) for container in docker_client.containers(): networks = container['NetworkSettings']['Networks'] if 'bridge' in networks: ip = networks['bridge']['IPAddress'] if ip: ip_to_containers[ip].append(container) output = [] for ip, containers in ip_to_containers.items(): if len(containers) > 1: output.append(f'{ip} shared by the following containers:') for container in containers: output.append(' Image: {}'.format(container['Image'])) output.append(' ID: {}'.format(container['Id'])) output.append(' State: {}'.format(container['State'])) output.append(' Status: {}'.format(container['Status'])) output.append('') if output: print( 'CRITICAL - There are multiple Docker containers assigned to the same IP.' ) print( 'There should only be one per IP. Choose one to keep and try stopping the others.' ) print('\n'.join(output)) return 2 else: print('OK - No Docker containers sharing an IP on this host.') targets_seen = {} duplicates_found = False for rule in list_docker_nat_rules(): target = rule.target_parameters if target not in targets_seen: targets_seen[target] = rule else: print( "This is the second time we've seen a rule with the same target_parameters!" ) print(rule) print("The other rule with that target is:") print(targets_seen[target]) duplicates_found = True if duplicates_found is True: print( "CRITICAL - Duplicate iptables rules found! This will route traffic to the wrong service!" ) return 2 else: print("OK - No duplicate Docker iptables rules detected")
def paasta_local_run(args): if args.action == 'build' and not makefile_responds_to('cook-image'): sys.stderr.write( "A local Makefile with a 'cook-image' target is required for --build\n" ) sys.stderr.write( "If you meant to pull the docker image from the registry, explicitly pass --pull\n" ) return 1 service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root) cluster = guess_cluster(service=service, args=args) instance = guess_instance(service=service, cluster=cluster, args=args) docker_client = get_docker_client() if args.action == 'build': default_tag = 'paasta-local-run-%s-%s' % (service, get_username()) tag = os.environ.get('DOCKER_TAG', default_tag) os.environ['DOCKER_TAG'] = tag pull_image = False cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root) if cook_return != 0: return cook_return elif args.action == 'dry_run': pull_image = False tag = None else: pull_image = True tag = None try: configure_and_run_docker_container( docker_client=docker_client, docker_hash=tag, service=service, instance=instance, cluster=cluster, args=args, pull_image=pull_image, dry_run=args.action == 'dry_run', ) except errors.APIError as e: sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e)) return 1
def main(): args = parse_args() docker_client = get_docker_client() running_mesos_task_ids = get_running_task_ids_from_mesos_slave() running_mesos_docker_containers = get_running_mesos_docker_containers(docker_client) print running_mesos_task_ids for container in running_mesos_docker_containers: mesos_task_id = mesos_tools.get_mesos_id_from_container(container=container, client=docker_client) print mesos_task_id if mesos_task_id not in running_mesos_task_ids: if args.force: print "Killing %s. (%s)" % (container["Names"][0], mesos_task_id) docker_client.kill(container) else: print "Would kill %s. (%s)" % (container["Names"][0], mesos_task_id) else: print "Not killing %s. (%s)" % (container["Names"][0], mesos_task_id)
def paasta_local_run(args): if args.pull or args.dry_run: build = False elif args.build: build = True else: build = local_makefile_present() service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root) cluster = guess_cluster(service=service, args=args) instance = guess_instance(service=service, cluster=cluster, args=args) docker_client = get_docker_client() if build: default_tag = 'paasta-local-run-%s-%s' % (service, get_username()) tag = os.environ.get('DOCKER_TAG', default_tag) os.environ['DOCKER_TAG'] = tag pull_image = False cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root) if cook_return != 0: return cook_return elif args.dry_run: pull_image = False tag = None else: pull_image = True tag = None try: configure_and_run_docker_container( docker_client=docker_client, docker_hash=tag, service=service, instance=instance, cluster=cluster, args=args, pull_image=pull_image, dry_run=args.dry_run, ) except errors.APIError as e: sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e)) return 1
def main(): if clog is None: print("CLog logger unavailable, exiting.", file=sys.stderr) sys.exit(1) clog.config.configure( scribe_host="169.254.255.254", scribe_port=1463, monk_disable=False, scribe_disable=False, ) cluster = load_system_paasta_config().get_cluster() client = get_docker_client() for ( timestamp, hostname, container_id, process_name, ) in capture_oom_events_from_stdin(): try: docker_inspect = client.inspect_container(resource_id=container_id) except (APIError): continue env_vars = get_container_env_as_dict(docker_inspect) service = env_vars.get("PAASTA_SERVICE", "unknown") instance = env_vars.get("PAASTA_INSTANCE", "unknown") mesos_container_id = env_vars.get("MESOS_CONTAINER_NAME", "mesos-null") mem_limit = env_vars.get("PAASTA_RESOURCE_MEM", "unknown") log_line = LogLine( timestamp=timestamp, hostname=hostname, container_id=container_id, cluster=cluster, service=service, instance=instance, process_name=process_name, mesos_container_id=mesos_container_id, mem_limit=mem_limit, ) log_to_clog(log_line) log_to_paasta(log_line) send_sfx_event(service, instance, cluster)
def paasta_local_run(args): if args.action == 'build' and not makefile_responds_to('cook-image'): sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n") sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n") return 1 service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root) cluster = guess_cluster(service=service, args=args) instance = guess_instance(service=service, cluster=cluster, args=args) docker_client = get_docker_client() if args.action == 'build': default_tag = 'paasta-local-run-%s-%s' % (service, get_username()) tag = os.environ.get('DOCKER_TAG', default_tag) os.environ['DOCKER_TAG'] = tag pull_image = False cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root) if cook_return != 0: return cook_return elif args.action == 'dry_run': pull_image = False tag = None else: pull_image = True tag = None try: configure_and_run_docker_container( docker_client=docker_client, docker_hash=tag, service=service, instance=instance, cluster=cluster, args=args, pull_image=pull_image, dry_run=args.action == 'dry_run', ) except errors.APIError as e: sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e)) return 1
def main(): scribe_logger = ScribeLogger(host='169.254.255.254', port=1463, retry_interval=5) cluster = load_system_paasta_config().get_cluster() client = get_docker_client() for timestamp, hostname, container_id, process_name in capture_oom_events_from_stdin(): try: docker_inspect = client.inspect_container(resource_id=container_id) except (APIError): continue env_vars = get_container_env_as_dict(docker_inspect) service = env_vars.get('PAASTA_SERVICE', 'unknown') instance = env_vars.get('PAASTA_INSTANCE', 'unknown') log_line = LogLine( timestamp=timestamp, hostname=hostname, container_id=container_id, cluster=cluster, service=service, instance=instance, process_name=process_name, ) log_to_scribe(scribe_logger, log_line) log_to_paasta(log_line) send_sfx_event(service, instance, cluster)
def docker_is_available(context): docker_client = get_docker_client() assert docker_client.ping() context.docker_client = docker_client
def main(): docker_client = get_docker_client() kill_containers_with_duplicate_iptables_rules(docker_client)
def paasta_local_run(args): if args.action == "pull" and os.geteuid() != 0 and not docker_config_available(): paasta_print("Re-executing paasta local-run --pull with sudo..") os.execvp("sudo", ["sudo", "-H"] + sys.argv) if args.action == "build" and not makefile_responds_to("cook-image"): paasta_print( "A local Makefile with a 'cook-image' target is required for --build", file=sys.stderr, ) paasta_print( "If you meant to pull the docker image from the registry, explicitly pass --pull", file=sys.stderr, ) return 1 try: system_paasta_config = load_system_paasta_config() except PaastaNotConfiguredError: paasta_print( PaastaColors.yellow( "Warning: Couldn't load config files from '/etc/paasta'. This indicates" "PaaSTA is not configured locally on this host, and local-run may not behave" "the same way it would behave on a server configured for PaaSTA." ), sep="\n", ) system_paasta_config = SystemPaastaConfig({"volumes": []}, "/etc/paasta") local_run_config = system_paasta_config.get_local_run_config() service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root) if args.cluster: cluster = args.cluster else: try: cluster = local_run_config["default_cluster"] except KeyError: paasta_print( PaastaColors.red( "PaaSTA on this machine has not been configured with a default cluster." "Please pass one to local-run using '-c'." ), sep="\n", file=sys.stderr, ) return 1 instance = args.instance docker_client = get_docker_client() docker_sha = None docker_url = None if args.action == "build": default_tag = "paasta-local-run-{}-{}".format(service, get_username()) docker_url = os.environ.get("DOCKER_TAG", default_tag) os.environ["DOCKER_TAG"] = docker_url pull_image = False cook_return = paasta_cook_image( args=None, service=service, soa_dir=args.yelpsoa_config_root ) if cook_return != 0: return cook_return elif args.action == "dry_run": pull_image = False docker_url = None docker_sha = args.sha else: pull_image = True docker_url = None docker_sha = args.sha try: return configure_and_run_docker_container( docker_client=docker_client, docker_url=docker_url, docker_sha=docker_sha, service=service, instance=instance, cluster=cluster, args=args, pull_image=pull_image, system_paasta_config=system_paasta_config, dry_run=args.action == "dry_run", ) except errors.APIError as e: paasta_print("Can't run Docker container. Error: %s" % str(e), file=sys.stderr) return 1
def main() -> None: parser = argparse.ArgumentParser( description=( 'Script to dump a HAProxy map between container IPs and task IDs.' ), ) parser.add_argument( '--update-haproxy', '-U', action='store_true', help='Whether to update haproxy for map updates', ) parser.add_argument( '--haproxy-timeout', '-T', type=int, default=1, help='Timeout for haproxy socket connections', ) parser.add_argument( 'map_file', nargs='?', default='/var/run/synapse/maps/ip_to_service.map', help='Where to write the output map file', ) args = parser.parse_args() if args.update_haproxy: prev_ip_to_task_id = get_prev_file_contents(args.map_file) new_lines = [] ip_addrs = [] service_ips_and_ids = extract_taskid_and_ip(get_docker_client()) for ip_addr, task_id in service_ips_and_ids: ip_addrs.append(ip_addr) if args.update_haproxy: update_haproxy_mapping( ip_addr, task_id, prev_ip_to_task_id, args.map_file, args.haproxy_timeout, ) new_lines.append('{ip_addr} {task_id}'.format( ip_addr=ip_addr, task_id=task_id, ) ) if args.update_haproxy: remove_stopped_container_entries( prev_ip_to_task_id.keys(), ip_addrs, args.map_file, args.haproxy_timeout, ) # Replace the file contents with the new map with atomic_file_write(args.map_file) as fp: fp.write('\n'.join(new_lines))
def paasta_local_run(args): if args.action == 'build' and not makefile_responds_to('cook-image'): paasta_print("A local Makefile with a 'cook-image' target is required for --build", file=sys.stderr) paasta_print("If you meant to pull the docker image from the registry, explicitly pass --pull", file=sys.stderr) return 1 try: system_paasta_config = load_system_paasta_config() except PaastaNotConfiguredError: paasta_print( PaastaColors.yellow( "Warning: Couldn't load config files from '/etc/paasta'. This indicates" "PaaSTA is not configured locally on this host, and local-run may not behave" "the same way it would behave on a server configured for PaaSTA." ), sep='\n', ) system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta') local_run_config = system_paasta_config.get_local_run_config() service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root) if args.cluster: cluster = args.cluster else: try: cluster = local_run_config['default_cluster'] except KeyError: paasta_print( PaastaColors.red( "PaaSTA on this machine has not been configured with a default cluster." "Please pass one to local-run using '-c'."), sep='\n', file=sys.stderr, ) return 1 instance = args.instance docker_client = get_docker_client() if args.action == 'build': default_tag = 'paasta-local-run-%s-%s' % (service, get_username()) tag = os.environ.get('DOCKER_TAG', default_tag) os.environ['DOCKER_TAG'] = tag pull_image = False cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root) if cook_return != 0: return cook_return elif args.action == 'dry_run': pull_image = False tag = None else: pull_image = True tag = None try: return configure_and_run_docker_container( docker_client=docker_client, docker_hash=tag, service=service, instance=instance, cluster=cluster, args=args, pull_image=pull_image, system_paasta_config=system_paasta_config, dry_run=args.action == 'dry_run', ) except errors.APIError as e: paasta_print( 'Can\'t run Docker container. Error: %s' % str(e), file=sys.stderr, ) return 1
def paasta_local_run(args): if args.action == 'build' and not makefile_responds_to('cook-image'): sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n") sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n") return 1 try: system_paasta_config = load_system_paasta_config() except PaastaNotConfiguredError: sys.stdout.write(PaastaColors.yellow( "Warning: Couldn't load config files from '/etc/paasta'. This indicates\n" "PaaSTA is not configured locally on this host, and local-run may not behave\n" "the same way it would behave on a server configured for PaaSTA.\n" )) system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta') local_run_config = system_paasta_config.get_local_run_config() service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root) if args.cluster: cluster = args.cluster else: try: cluster = local_run_config['default_cluster'] except KeyError: sys.stderr.write(PaastaColors.red( "PaaSTA on this machine has not been configured with a default cluster.\n" "Please pass one to local-run using '-c'.\n")) return 1 instance = args.instance docker_client = get_docker_client() if args.action == 'build': default_tag = 'paasta-local-run-%s-%s' % (service, get_username()) tag = os.environ.get('DOCKER_TAG', default_tag) os.environ['DOCKER_TAG'] = tag pull_image = False cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root) if cook_return != 0: return cook_return elif args.action == 'dry_run': pull_image = False tag = None else: pull_image = True tag = None try: configure_and_run_docker_container( docker_client=docker_client, docker_hash=tag, service=service, instance=instance, cluster=cluster, args=args, pull_image=pull_image, system_paasta_config=system_paasta_config, dry_run=args.action == 'dry_run', ) except errors.APIError as e: sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e)) return 1