def build_and_run(options, request):
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
start_time = str(int(unix_time_milliseconds(datetime.utcnow())))
run_containers(cmd, options)
def fin():
# Stop the containers then remove them and their volumes (--volumes option)
print("containers stopping", flush=True)
try:
# Used for when there are multiple filewriter instances
# as the service is not called "filewriter"
multiple_log_options = dict(options)
multiple_log_options["SERVICE"] = ["filewriter1", "filewriter2"]
cmd.logs(multiple_log_options)
except:
log_options = dict(options)
log_options["SERVICE"] = ["filewriter"]
cmd.logs(log_options)
options["--timeout"] = 30
cmd.down(options)
print("containers stopped", flush=True)
# Using a finalizer rather than yield in the fixture means
# that the containers will be brought down even if tests fail
request.addfinalizer(fin)
# Return the start time so the filewriter knows when to start consuming data
# from to get all data which was published
return start_time
def up(project_path):
up_options = {
"-d": True,
"--no-color": False,
"--no-deps": False,
"--build": False,
"--abort-on-container-exit": False,
"--remove-orphans": False,
"--no-recreate": True,
"--force-recreate": False,
"--no-build": False,
"SERVICE": "",
"--scale": []
}
project = project_from_options(project_path, up_options)
cmd = TopLevelCommand(project)
cmd.up(up_options)
ps_options = {"SERVICE": "", "-q": True}
containers = sorted(
project.containers(service_names=ps_options['SERVICE'], stopped=True) +
project.containers(service_names=ps_options['SERVICE'],
one_off=OneOffFilter.only),
key=attrgetter('name'))
container_ids = []
for container in containers:
container_ids.append(container.id)
return container_ids
def __init__(self, path='./'):
path = './' # Path to docker-compose directory
self.options = {
"--file": ['docker-compose.dev.yaml', ],
"--no-deps": False,
"--abort-on-container-exit": False,
"SERVICE": "",
"--remove-orphans": False,
"--no-recreate": True,
"--force-recreate": False,
"--build": False,
'--no-build': False,
'--no-color': False,
"--rmi": "none",
"--volumes": "",
"--follow": False,
"--timestamps": False,
"--tail": "all",
"-d": True,
'--always-recreate-deps': False,
'--scale': []
}
self.project = project_from_options(path, self.options)
self.cli = TopLevelCommand(self.project)
def docker_compose(request):
"""
:type request: _pytest.python.FixtureRequest
"""
options = {"--no-deps": False,
"--abort-on-container-exit": False,
"SERVICE": "",
"--remove-orphans": False,
"--no-recreate": True,
"--force-recreate": False,
"--build": False,
'--no-build': False,
'--no-color': False,
"--rmi": "none",
"--volumes": "",
"--follow": False,
"--timestamps": False,
"--tail": "all",
"-d": True,
}
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
cmd.up(options)
def fin():
cmd.logs(options)
cmd.down(options)
request.addfinalizer(fin)
def rm(ctx, clustername, volumes):
cluster_dir = os.path.join(ctx.obj['etc'], clustername)
if not os.path.exists(cluster_dir):
click.echo(click.style('wrong cluster name', fg='red', bold=True))
ctx.exit(-1)
project = project_from_options(cluster_dir, {})
project.down(remove_image_type=0, include_volumes=volumes)
ensure_dir_absent(cluster_dir)
click.echo(
click.style('cluster {} removed'.format(clustername), fg='green'))
def __init__(self, docker_compose_file, options=None):
self.docker_compose_file = docker_compose_file
# build docker options with default ones + input overrides
dockerenv_options = self.options
if options:
dockerenv_options.update(options)
project = project_from_options(str(TESTS_DIR), dockerenv_options)
self.cmd = TopLevelCommand(project)
self.cmd.up(dockerenv_options)
def before(self, *args, **kwargs):
"""
Will run `docker-compose up -d` only in case docker-compose is installed locally
and there is a docker-compose.yml file in resources directory
"""
compose = self.find_resource_file()
if compose and is_package_installed('compose'):
from compose.cli.main import TopLevelCommand, project_from_options
info('Starting docker-compose. Please wait.')
self._options['-f'] = join(self._resources, compose),
self._cmd = TopLevelCommand(
project_from_options(self._resources, self._options))
self._cmd.up(self._options)
def ps_one_project(ctx, cluster_name):
cluster_dir = os.path.join(ctx.obj['etc'], cluster_name)
if not os.path.exists(cluster_dir):
click.echo(click.style('wrong cluster name', fg='red', bold=True))
ctx.exit(-1)
project = project_from_options(cluster_dir, {})
header, body = ['service type', 'service address'], []
# 其实都只有一个
tidb_container = [
c for s in project.services if 'tidb' in s.name
for c in s.containers()
][0]
tidb_port = tidb_container.inspect()['NetworkSettings']['Ports']
tidb_address = 'localhost:{}'.format(tidb_port['4000/tcp'][0].get(
'HostPort', 'unknown'))
grafana_container = [
c for s in project.services if 'grafana' in s.name
for c in s.containers()
][0]
grafana_port = grafana_container.inspect()['NetworkSettings']['Ports']
grafana_address = 'localhost:{}'.format(grafana_port['3000/tcp'][0].get(
'HostPort', 'unknown'))
service_info = tabulate([
('tidb', tidb_address),
('grafana', grafana_address),
],
headers=['service type', 'service address'])
body = []
for service in project.services:
containers = service.containers()
containers_status = [c.inspect() for c in containers]
running = len([
c for c in containers_status if c['State']['Status'] == 'running'
])
body.append(
(service.name,
'{}/{} running/total containers'.format(running,
len(containers_status))))
service_status = tabulate(body,
headers=['service name', 'container status'])
click.echo(
tabulate(
[(cluster_name, service_info, service_status)],
['cluster name', 'service info', 'service status'],
tablefmt='fancy_grid',
))
def parse_compose_project(project_dir, machine_name, name, compose_files):
""" Parse the compose files """
machine_options = get_machine_options(machine_name)
compose_options = {
**machine_options,
"--file": compose_files,
"--project-name": name,
"--project-directory": project_dir,
}
# out = io.StringIO()
# with redirect_stderr(out):
project = project_from_options(project_dir, compose_options)
return project
def rm(project_path):
rm_options = {
"--force": True,
"--stop": True,
"-v": False,
"--rmi": "none",
"--volumes": "/private",
"--remove-orphans": False,
"SERVICE": ""
}
project = project_from_options(project_path, rm_options)
cmd = TopLevelCommand(project)
cmd.down(rm_options)
def start_ioc(request):
options = common_options
options["--project-name"] = "ioc"
options["--file"] = [os.path.join("compose", "docker-compose-ioc.yml")]
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
cmd.up(options)
def fin():
print("Stopping zookeeper and kafka", flush=True)
options["--timeout"] = 30
options["--project-name"] = "ioc"
options["--file"] = [os.path.join("compose", "docker-compose-ioc.yml")]
cmd.down(options)
request.addfinalizer(fin)
def get_container_id(container_name):
options = {
"--quiet": True,
"--filter": "status=running",
"nginx": True,
"--services": False,
"SERVICE": [container_name],
}
project = project_from_options(os.path.dirname(os.path.abspath(__file__)),
options)
cmd = TopLevelCommand(project)
result = io.StringIO()
with redirect_stdout(result):
cmd.ps(options)
return result.getvalue()
def build_and_run(options, request):
build_forwarder_image()
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
run_containers(cmd, options)
def fin():
# Stop the containers then remove them and their volumes (--volumes option)
print("containers stopping", flush=True)
log_options = dict(options)
log_options["SERVICE"] = ["forwarder"]
cmd.logs(log_options)
options["--timeout"] = 30
cmd.down(options)
print("containers stopped", flush=True)
# Using a finalizer rather than yield in the fixture means
# that the containers will be brought down even if tests fail
request.addfinalizer(fin)
def start_kafka(request):
print("Starting zookeeper and kafka", flush=True)
options = common_options
options["--project-name"] = "kafka"
options["--file"] = ["docker-compose-kafka.yml"]
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
cmd.up(options)
print("Started kafka containers", flush=True)
wait_until_kafka_ready(cmd, options)
def fin():
print("Stopping zookeeper and kafka", flush=True)
options["--timeout"] = 30
options["--project-name"] = "kafka"
options["--file"] = ["docker-compose-kafka.yml"]
cmd.down(options)
request.addfinalizer(fin)
def build_and_run(options, request):
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
run_containers(cmd, options)
start_time = str(int(unix_time_milliseconds(datetime.utcnow())))
def fin():
# Stop the containers then remove them and their volumes (--volumes option)
print("containers stopping", flush=True)
log_options = dict(options)
cmd.logs(log_options)
options["--timeout"] = 30
cmd.down(options)
print("containers stopped", flush=True)
# Using a finalizer rather than yield in the fixture means
# that the containers will be brought down even if tests fail
request.addfinalizer(fin)
return start_time
def dao_2048(service_name, config_file):
options = { "--no-deps":False,
"--always-recreate-deps":False,
"--abort-on-container-exit": False,
"SERVICE": service_name,
"--remove-orphans": False,
"--no-recreate": True,
"--force-recreate": False,
"--build": False,
'--no-build': False,
'--no-color': False,
"--rmi": "none",
"--volumes": "",
"--follow": False,
"--timestamps": False,
"--tail": "all",
"-d": True,
"--scale": {'master=1'},
"--file": config_file
}
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
cmd.up(options)
def docker_services(docker_compose_file, docker_allow_fallback,
docker_compose_project_name, docker_compose_project_dir):
"""Ensure all Docker-based services are up and running."""
docker_compose = DockerComposeExecutor(docker_compose_file,
docker_compose_project_name,
docker_compose_project_dir)
# If we allowed to run without Docker, check it's presence
if docker_allow_fallback is True:
try:
with open(os.devnull, 'w') as devnull:
subprocess.call(['docker', 'ps'],
stdout=devnull,
stderr=devnull)
except Exception:
yield Services(None, docker_allow_fallback=True)
return
project = project_from_options(docker_compose._compose_project_dir,
options=docker_compose.as_dict_options())
cmd = TopLevelCommand(project)
# Spawn containers.
up_options = docker_compose.defaults_opts('up')
up_options['-d'] = True
up_options['--build'] = True
cmd.up(up_options)
# Let test(s) run.
yield Services(project)
# Clean up.
down_option = docker_compose.defaults_opts('down')
down_option['-v'] = True
cmd.down(down_option)
def test_environment(request):
"""
:type request: _pytest.python.FixtureRequest
"""
# Options must be given as long form
options = common_options
options["--file"] = ["docker-compose.yml"]
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
print("Running docker-compose up", flush=True)
cmd.pull(options)
cmd.up(options)
print("\nFinished docker-compose up\n", flush=True)
wait_until_kafka_ready(cmd, options)
def fin():
cmd.logs(options)
# Stop the containers then remove them and their volumes (--volumes option)
cmd.down(options)
# Using a finalizer rather than yield in the fixture means
# that the containers will be brought down even if tests fail
request.addfinalizer(fin)
def __init__(self, yml_dir):
self._output = io.StringIO()
self.path_to_yml_dir = yml_dir
self.options = GENERAL_COMPOSE_OPTIONS
self.project = project_from_options(self.path_to_yml_dir, self.options)
self._cmd = TopLevelCommand(self.project)
def top_level_command_from_options(self, options):
"""Creates a top level command from options
:param options (str) docker compose options"""
project = project_from_options(self.project_location, options)
cmd = TopLevelCommand(project)
return cmd
def create_cluster(ctx, base, network, cluster_name, pd_count, tikv_count,
tidb_version):
cluster_dir = os.path.join(ctx.obj['etc'], cluster_name)
# config
pdservices = [{
'name': 'pd_{}_{}'.format(cluster_name, index),
'base': os.path.abspath(base),
'image': 'pingcap/pd:latest',
} for index in range(pd_count)]
tikvservices = [{
'name': 'tikv_{}_{}'.format(cluster_name, index),
'base': os.path.abspath(base),
'image': 'pingcap/tikv:latest',
} for index in range(tikv_count)]
tidb = {
'base': os.path.abspath(base),
'image': 'pingcap/tidb:{}'.format(tidb_version),
}
monitor = {'base': os.path.abspath(base)}
# ensure dir exists
ensure_dir(base)
ensure_dir(os.path.join(base, 'data'))
ensure_dir(os.path.join(base, 'logs'))
ensure_dir(cluster_dir)
# create config
shutil.copytree('./config', os.path.join(base, 'config'))
# edit some configs
filenames = [
os.path.join(
base, 'config/grafana/provisioning/datasources/datasources.yaml')
]
massedit.edit_files(filenames, [
"re.sub(r'prometheus:9090', 'prometheus_{}:9090', line)".format(
cluster_name)
],
dry_run=False)
filenames = [os.path.join(base, 'prometheus.yml')]
massedit.edit_files(filenames, [
"re.sub(r'pushgateway:9091', 'pushgateway_{}:9091', line)".format(
cluster_name)
],
dry_run=False)
# keep docker-compose.yml
tp = Jinja2(__name__)
content = tp.render_template('/docker_compose.jinja',
network=network,
pdservices=pdservices,
tikvservices=tikvservices,
tidb=tidb,
monitor=monitor,
cluster_name=cluster_name)
filename = os.path.join(cluster_dir, 'docker-compose.yml')
with open(filename, 'w') as f:
f.write(content)
project = project_from_options(cluster_dir, {})
project.up(detached=True)
click.echo(
click.style('cluster {} created'.format(cluster_name), fg='green'))
def __init__(self, docker_compose_file):
self.docker_compose_file = docker_compose_file
project = project_from_options(os.path.dirname(__file__), self.options)
self.cmd = TopLevelCommand(project)
self.cmd.up(self.options)
def build_and_run(options, request, local_path=None, wait_for_debugger=False):
if wait_for_debugger and local_path is None:
warnings.warn(
"Option specified to wait for debugger to attach, but this "
"can only be used if a local build path is provided")
project = project_from_options(os.path.dirname(__file__), options)
cmd = TopLevelCommand(project)
start_time = str(int(unix_time_milliseconds(datetime.utcnow())))
run_containers(cmd, options)
if local_path is not None:
# Launch local build of file writer
full_path_of_file_writer_exe = os.path.join(local_path, "bin",
"kafka-to-nexus")
log_file = open("logs/file-writer-logs.txt", "w")
proc = Popen(
[
full_path_of_file_writer_exe,
"-c",
"./config-files/local_file_writer_config.ini",
],
stdout=log_file,
)
if wait_for_debugger:
proc.send_signal(
signal.SIGSTOP
) # Pause the file writer until we've had chance to attach a debugger
input(
f"\n"
f"Attach a debugger to process id {proc.pid} now if you wish, then press enter to continue: "
)
print(
"You'll need to tell the debugger to continue after it has attached, "
'for example type "continue" if using gdb.')
proc.send_signal(signal.SIGCONT)
def fin():
# Stop the containers then remove them and their volumes (--volumes option)
print("containers stopping", flush=True)
if local_path is None:
try:
# Used for when there are multiple filewriter instances
# as the service is not called "filewriter"
multiple_log_options = dict(options)
multiple_log_options["SERVICE"] = [
"filewriter1", "filewriter2"
]
cmd.logs(multiple_log_options)
except:
log_options = dict(options)
log_options["SERVICE"] = ["filewriter"]
cmd.logs(log_options)
else:
proc.kill()
options["--timeout"] = 30
cmd.down(options)
print("containers stopped", flush=True)
# Using a finalizer rather than yield in the fixture means
# that the containers will be brought down even if tests fail
request.addfinalizer(fin)
# Return the start time so the filewriter knows when to start consuming data
# from to get all data which was published
return start_time
