def load_image_from_file(name):
from docker import DockerClient
docker = DockerClient.from_env(version="auto")
with open(name, "rb") as f:
images = docker.images.load(f.read())
image = images[0]
return image.tags[0]
def find_and_stop(cls, service_name):
docker = DockerClient.from_env(version="auto")
for container in docker.containers.list(all=True):
if container.name.endswith(".{}".format(service_name)):
container.stop()
container.remove(force=True)
docker.networks.get(service_name).remove()
def __init__(self, name, network, image, bolt_address, http_address, auth, **config):
self.name = name
self.network = network
self.image = "{}:{}".format(self.repository, image)
self.bolt_address = bolt_address
self.http_address = http_address
self.auth = auth
self.docker = DockerClient.from_env()
environment = {
"NEO4J_AUTH": "/".join(self.auth),
"NEO4J_ACCEPT_LICENSE_AGREEMENT": "yes",
}
for key, value in config.items():
environment["NEO4J_" + key.replace("_", "__").replace(".", "_")] = value
ports = {
"7474/tcp": self.http_address,
"7687/tcp": self.bolt_address,
}
self.container = self.docker.containers.create(self.image,
detach=True,
environment=environment,
hostname="{}.{}".format(self.name, self.network.name),
name="{}.{}".format(self.name, self.network.name),
network=self.network.name,
ports=ports)
self.ip_address = None
def __init__(self,
name=None,
image=None,
auth=None,
n_cores=None,
n_replicas=None,
bolt_port=None,
http_port=None,
https_port=None,
debug_port=None,
debug_suspend=None,
dir_spec=None,
config=None,
env=None):
from docker import DockerClient
self.name = name or self._random_name()
self.docker = DockerClient.from_env(version="auto")
self.image = resolve_image(image or self.default_image)
self.auth = Auth(*auth) if auth else make_auth()
if self.auth.user != "neo4j":
raise ValueError("Auth user must be 'neo4j' or empty")
self.machines = {}
self.network = None
self.routing_tables = {"system": Neo4jRoutingTable()}
self.console = None
def docker_client():
# todo improve when yellowbox is upgraded
try:
ret = DockerClient.from_env()
ret.ping()
except Exception:
return DockerClient(base_url="tcp://localhost:2375")
else:
return ret
def __init__(self, name, **parameters):
self.docker = DockerClient.from_env()
self.image = self.fix_image(parameters.get("image"))
self.user = parameters.get("user", "neo4j")
self.password = parameters.get("password", "password")
self.network = self.docker.networks.create(name)
self.machines = []
self.routers = []
self.bolt_port_range = range(17600, 17700)
self.http_port_range = range(17400, 17500)
def __init__(self,
environment,
instance_id,
host: str,
cert_path: str,
tls_verify: bool = True,
compose_tls_version: str = 'TLSv1_2',
version: str = 'auto'):
""" Run the super constructor but also set class properties
In order to decorate this existing class as a DockerClient, without using the
DockerClient constructor, nor the from_env class method, we have to reproduce
the APIClient construction.
We could rewrite some of the DockerClient functionality, or we can just create
a throwaway DockerClient instance and steal its ApiClient and use it.
This also lets us easily include any other env variables that might be in
scope.
Parameters:
-----------
All of the arguments are handed off to the DockerClient.from_env class method.
All arguments are converted to the related ENV equivalent if it had been set
outside of this python code.
host (str) [DOCKER_HOST] what daemon host to use (docker socket)
cert_path (str) [DOCKER_CERT_PATH] path to rsa keys for authorization
tls_verify (bool) [DOCKER_TLS_VERIFY] should the client pursue TLS verification
compose_tls_version (str) [COMPOSE_TLS_VERSION] what TLS version should
the Docker client use for docker compose.
"""
super(ClientBase, self).__init__(environment, instance_id)
logger.debug(
"Configuring docker client with args for host:{}".format(host))
self.host = host
self.cert_path = cert_path
self.tls_verify = '1' if tls_verify else '0'
self.compose_tls_version = compose_tls_version
env = os.environ.copy()
env['DOCKER_HOST'] = self.host
env['DOCKER_CERT_PATH'] = self.cert_path
env['DOCKER_TLS_VERIFY'] = self.tls_verify
env['COMPOSE_TLS_VERSION'] = self.compose_tls_version
throwaway = DockerClient.from_env(environment=env, version=version)
self.api = throwaway.api
def _connect(self):
try:
Log.i("Attempt to contact Docker Daemon")
self._change_status(ManagerStatus.ATTEMPT_CONNECT)
self._client = DockerClient.from_env(**self._kwargs)
self._last_ping = datetime.now()
Log.i("Connected to Docker Daemon")
self._change_status(ManagerStatus.CONNECTED)
self.refresh_all()
except DockerException as e:
Log.e(e)
self._stop = True
self._change_status(ManagerStatus.DISCONNECTED)
def docker_client():
"""
Starts docker client from the environment, with a fallback to default TCP port
(for running from within virtual machines)
"""
try:
ret = DockerClient.from_env()
ret.ping()
except Exception: # pragma: no cover
ret = DockerClient(base_url='tcp://localhost:2375')
ret.ping()
with closing(ret):
yield ret
def __init__(self):
base_url = Config.get("docker_base_url", None)
version = Config.get("docker_version", None)
tls = Config.get("docker_tls", None)
if base_url is None:
self.docker_cli = DockerClient.from_env()
else:
self.docker_cli = DockerClient(base_url=base_url,
version=version,
tls=tls)
self.thread_count = Config.get("thread_count")
self.image_info: self.__Image = None
self.image = None
self.container = None
def __init__(self, spec, image, auth):
from docker import DockerClient
from docker.errors import ImageNotFound
self.spec = spec
self.image = image
self.address = Address(("localhost", self.spec.bolt_port))
self.addresses = AddressList([("localhost", self.spec.bolt_port)])
self.auth = auth
self.docker = DockerClient.from_env(version="auto")
environment = {}
if self.auth:
environment["NEO4J_AUTH"] = "/".join(self.auth)
environment["NEO4J_ACCEPT_LICENSE_AGREEMENT"] = "yes"
for key, value in self.spec.config.items():
fixed_key = "NEO4J_" + key.replace("_", "__").replace(".", "_")
environment[fixed_key] = value
for key, value in self.spec.env.items():
environment[key] = value
ports = {
"7474/tcp": self.spec.http_port,
"7473/tcp": self.spec.https_port,
"7687/tcp": self.spec.bolt_port,
}
if self.spec.debug_opts.port is not None:
ports["5100/tcp"] = self.spec.debug_opts.port
if self.spec.dir_spec:
volumes = self.spec.dir_spec.volumes(self.spec.name)
for path in volumes:
makedirs(path, exist_ok=True)
else:
volumes = None
def create_container(img):
return self.docker.containers.create(
img,
detach=True,
environment=environment,
hostname=self.spec.fq_name,
name=self.spec.fq_name,
network=self.spec.service_name,
ports=ports,
volumes=volumes,
)
try:
self.container = create_container(self.image)
except ImageNotFound:
log.info("Downloading Docker image %r", self.image)
self.docker.images.pull(self.image)
self.container = create_container(self.image)
def download_snapshot_artifact(artifact):
from docker import DockerClient
docker = DockerClient.from_env(version="auto")
log.info("Downloading {} from «{}»".format(
artifact, snapshot_host))
url = "{}/{}".format(snapshot_build_url, artifact)
r = teamcity_http.request("GET", url)
if r.status == 200:
images = docker.images.load(r.data)
image = images[0]
return image.tags[0]
else:
log.error("{}".format(r.data))
raise RuntimeError("Download failed with HTTP "
"status {} {}".format(r.status, r.reason))
def pull_snapshot(edition, force):
""" Ensure a local copy of the snapshot image is available. If 'force' is
True, then a download will always happen, regardless of the local cache.
"""
from docker import DockerClient
from docker.errors import ImageNotFound
docker = DockerClient.from_env(version="auto")
artifact = resolve_artifact_name(edition)
if force:
return download_snapshot_artifact(artifact)
else:
derived = derive_image_tag(artifact)
try:
docker.images.get(derived)
except ImageNotFound:
return download_snapshot_artifact(artifact)
else:
return derived
def __init__(self, name=None, image=None, auth=None, **parameters):
self.name = name or uuid4().hex[-7:]
self.docker = DockerClient.from_env(version="auto")
headers = {}
if TEAMCITY_USER and TEAMCITY_PASSWORD:
headers.update(
make_headers(basic_auth="{}:{}".format(TEAMCITY_USER,
TEAMCITY_PASSWORD)))
self.http = PoolManager(
cert_reqs="CERT_REQUIRED",
ca_certs=certifi.where(),
headers=headers,
)
self.image = self._resolve_image(image)
self.auth = auth or make_auth()
self.machines = []
self.routers = []
self.network = None
def init_env(self, timeout=120, version='auto', env=None):
kwargs = {'timeout': timeout, 'version': version, 'environment': env}
try:
self._client = DockerClient.from_env(**kwargs)
version = self._client.api.version()
DebugConsole.println('Environment loaded for %s' %
env['DOCKER_HOST'])
Log.i(
'Initialized communication using Docker Engine API version %s'
% version['ApiVersion'])
self._change_status(ManagerStatus.CONNECTED)
self.refresh_all()
Log.i('Loaded environment')
except DockerException as e:
print("DockerManager :: %s" % e)
self._change_status(ManagerStatus.DISCONNECTED)
DebugConsole.println('Docker Client could not load environment')
Log.e('Docker Client could not load environment')
def __init__(
self,
username: str,
password: str,
host="localhost",
port=4001,
trading_mode="paper",
start=False,
logger=None,
):
self.username = username
self.password = password
self.trading_mode = trading_mode
self.host = host
self.port = port
self._docker: DockerClient = DockerClient.from_env()
self._client: Optional[IB] = None
self._container = None
self.log = logger or logging.getLogger("nautilus_trader")
if start:
self.start()
def __init__(self, name, service_name, image, auth, bolt_port, http_port,
**config):
self.name = name
self.service_name = service_name
self.fq_name = "{}.{}".format(self.name, self.service_name)
self.image = image
self.bolt_port = bolt_port
self.http_port = http_port
self.addresses = AddressList([("localhost", self.bolt_port)])
self.auth = auth
self.docker = DockerClient.from_env(version="auto")
environment = {}
if self.auth:
environment["NEO4J_AUTH"] = "{}/{}".format(self.auth[0],
self.auth[1])
if "enterprise" in image:
environment["NEO4J_ACCEPT_LICENSE_AGREEMENT"] = "yes"
for key, value in config.items():
environment["NEO4J_" +
key.replace("_", "__").replace(".", "_")] = value
ports = {
"7474/tcp": self.http_port,
"7687/tcp": self.bolt_port,
}
def create_container(img):
return self.docker.containers.create(img,
detach=True,
environment=environment,
hostname=self.fq_name,
name=self.fq_name,
network=self.service_name,
ports=ports)
try:
self.container = create_container(self.image)
except ImageNotFound:
log.info("Downloading Docker image %r", self.image)
self.docker.images.pull(self.image)
self.container = create_container(self.image)
def docker_client() -> DockerClient:
client = DockerClient.from_env()
client.ping() # Make sure we're actually connected.
with closing(client):
yield client
docker_py.containers.run(
f'{IMAGE}:{TAG}',
environment=envs,
volumes=vols,
auto_remove=True,
# shm_size='2G',
# mem_limit='1g',
tty=True,
remove=True)
else:
docker_py.containers.run(
f'{IMAGE}:{TAG}',
environment=envs,
volumes=vols,
auto_remove=True,
# shm_size='2G',
# mem_limit='1g',
tty=True,
remove=True)
docker_py = DockerClient.from_env()
build_image()
push_image()
generate_printer_list()
build_matrix()
from os import chmod, path
from random import choice
from shutil import rmtree
from tempfile import mkdtemp
from threading import Thread
from time import sleep
from docker import DockerClient
from docker.errors import APIError, ImageNotFound
from packaging.version import Version
from py2neo.compat import perf_counter
from py2neo.security import Auth, make_auth, install_certificate, install_private_key
from py2neo.wiring import Address, Wire
docker = DockerClient.from_env(version="auto")
log = getLogger(__name__)
def random_name(size):
return "".join(choice("bcdfghjklmnpqrstvwxz") for _ in range(size))
class Neo4jInstance(object):
""" A single Neo4j server instance, potentially part of a cluster.
"""
container = None
ip_address = None
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Parse incoming arguments.")
parser.add_argument(
"-s",
"--server",
dest="server",
default="http://raas.perciplex.com",
help="Server IP address",
)
args = parser.parse_args()
server_ip = args.server
# Instantiate docker client
docker_client = DockerClient.from_env()
# clear LED screen
LedMessage("").stop()
while True:
try:
response = requests.get(
server_ip + "/api/job/pop",
params={
"FLASK_PASS": FLASK_PASS,
"hardware": socket.gethostname()
},
)
response_status = response.status_code
print(response)
def docker_client():
client = DockerClient.from_env()
yield client
def init_env(self, timeout=120, version='auto', env=None):
kwargs = {'timeout': timeout, 'version': version, 'environment': env}
self._client = DockerClient.from_env(**kwargs)
print("Connected to Docker Daemon")
self._change_status(ManagerStatus.CONNECTED)
def docker_registry():
from docker import DockerClient
from pachypy.registry import DockerRegistryAdapter
return DockerRegistryAdapter(DockerClient.from_env())
def main(file: File):
utils.configure_loguru()
dc = DockerClient.from_env()
# Load benchmark matrix
matrix = load(file, Loader=Loader)
client = matrix['client']
output = matrix['output']
runs_per_setup = matrix['runs-per-setup']
defaults = matrix['defaults']
# Setup the Docker network.
tc = containers.create_tc_container(dc)
net = containers.create_network(dc)
# Results are written at the end.
xlsx_sheets = {}
for setup in matrix['setups']:
setup_name = list(setup.keys())[0]
logger.info(f"===== {setup_name} =====")
setup = utils.merge(defaults, setup[setup_name])
setups = expand_setup(setup, setup_name)
for setup in setups:
variant_times = DataFrame()
variant_name = utils.add_delay_postfix(setup_name, setup)
runs = range(1, runs_per_setup + 1)
for r in runs:
try:
logger.info(
f"===== {variant_name} ({r}/{runs_per_setup}) =====")
y, z = (None, None)
if 'Y' in setup['containers']:
y = containers.create_container('Y', setup, net, dc)
if 'Z' in setup['containers']:
z = containers.create_container('Z', setup, net, dc)
# An X node is mandatory.
x = containers.create_container('X', setup, net, dc)
# Capture stdout until X completes.
logger.info(
f"Waiting for container '{x.name}' (X) to finish.")
stdout = []
for line in x.attach(stream=True):
stdout.append(line)
# Process results
logger.info(
f"Container '{x.name}' (X) finished. Processing results."
)
if client == "httping":
(times,
stats) = httping.extract_httping_run_result(stdout)
elif client == "wrk":
(times, stats) = wrk.extract_wrk_run_result(stdout)
variant_times[f'run{r}'] = times
variant_times[f'run{r}-stats'] = stats
# Always perform cleanup
finally:
if y is not None:
y.kill()
if z is not None:
z.kill()
# Calculate variant-wide stats
variant_stats = np.zeros(min(len(variant_times['run1']), 4))
combined = np.array([])
means = np.zeros(runs_per_setup)
for (i, r) in enumerate(runs):
means[i] = variant_times[f'run{r}-stats'][1]
combined = np.append(combined, variant_times[f'run{r}'])
variant_stats[0] = means.mean()
variant_stats[1] = means.std()
variant_stats[2] = combined.mean()
variant_stats[3] = combined.std()
# Add variant-wide stats to variant sheet
variant_times['stats'] = variant_stats
xlsx_sheets[variant_name] = variant_times
# Write output to file
logger.info("Writing output")
with ExcelWriter(output) as w:
for (name, df) in xlsx_sheets.items():
df.to_excel(w, sheet_name=name)
# Cleanup
net.remove()
tc.kill()
def client(self) -> DockerClient: # pylint: disable=no-self-use
"""Docker client."""
return DockerClient.from_env()
def __init__(self, docker_cli: Optional[DockerClient] = None):
self.docker_cli = docker_cli or DockerClient.from_env()
from pathlib import Path
from starlette.responses import FileResponse, JSONResponse, PlainTextResponse
from starlette.staticfiles import StaticFiles
from starlette.websockets import WebSocketState
from urllib.parse import urlsplit, urlunsplit
VISIDATA_IMAGE = environ.get("VISIDATA_IMAGE") or "visidata"
DOCKER_HOST = urlsplit(parse_docker_host(environ.get("DOCKER_HOST")))
assert DOCKER_HOST.scheme in {"http", "https", "http+unix"}, \
f"DOCKER_HOST={DOCKER_HOST!r} uses an unsupported scheme"
frontend = Path(__file__).resolve().parent.parent / "frontend"
assert frontend.is_dir(), f"{frontend} is not a directory"
docker = DockerClient.from_env()
logger = logging.getLogger(__name__)
logging.basicConfig()
__all__ = [
"index", "vd", "create_container", "start_container",
"resize_container_tty", "attach_to_container", "frontend_public"
]
frontend_public = StaticFiles(directory=frontend / "public")
def index(request):
return FileResponse(frontend / "index.html")
def client(self): # type: () -> DockerClient
"""Docker client.""" # pylint: disable=no-self-use
return DockerClient.from_env()
def __init__(
self,
environment,
instance_id,
host: str,
cert_path: str,
tls_verify: bool = True,
compose_tls_version: str = "TLSv1_2",
version: str = "auto",
):
"""Set class properties.
In order to decorate this existing class as a DockerClient, without
using the DockerClient constructor, nor the from_env class method, we
have to reproduce the APIClient construction.
We could rewrite some of the DockerClient functionality, or we can just
create a throwaway DockerClient instance and steal its ApiClient and
use it.
This also lets us easily include any other env variables that might be
in scope.
@Note that we don't run the docker-py constructor as we build our own
client.
Parameters:
-----------
All of the arguments are handed off to the DockerClient.from_env class
method. All arguments are converted to the related ENV equivalent if it
had been set outside of this python code.
host (str) [DOCKER_HOST] what daemon host to use (docker socket)
cert_path (str) [DOCKER_CERT_PATH] path to rsa keys for authorization
tls_verify (bool) [DOCKER_TLS_VERIFY] should the client pursue TLS
verification
compose_tls_version (str) [COMPOSE_TLS_VERSION] what TLS version should
the Docker client use for docker compose.
"""
self._environment: Environment = environment
""" Environemnt in which this plugin exists """
self._instance_id: str = instance_id
""" Unique id for this plugin instance """
logger.debug("Configuring docker client with args for host: %s", host)
self.host = host
self.cert_path = cert_path
self.tls_verify = "1" if tls_verify else "0"
self.compose_tls_version = compose_tls_version
env = os.environ.copy()
env["DOCKER_HOST"] = self.host
env["DOCKER_CERT_PATH"] = self.cert_path
env["DOCKER_TLS_VERIFY"] = self.tls_verify
env["COMPOSE_TLS_VERSION"] = self.compose_tls_version
# Build a client in the classical way, but we just take it's api and
# throw the client away
throwaway = DockerClient.from_env(environment=env, version=version)
self.api = throwaway.api
def _get_client(env=True, url=None):
if env:
return DockerClient.from_env()
return DockerClient(base_url=url)
