def update_container_info(self):
"""
update_container_info tries to start all docker containers a site is associated with:
The server where these dockers reside, depends on the options selected.
It could be either localhost, or the remote host.
Either two or three containers are handeled on each site:
- db: this is the container containing the database.
it is only checkd for existence and started when stopped.
- $DOCKERNAME: This is the docker that containes the actual site
The value of $DOCKERNAME is read from the site info using the key 'docker'
If the site is a slave, and a transfer from the master to the slave is requested:
- $MASTER_DOCKERNAME: this is the container name of the master site as found in sites.py.
"""
name = self.site_name
site_info = self.sites[name]
erp_provider = site_info.get('erp_provider')
docker = site_info.get('docker')
if not docker or not docker.get('container_name'):
print(
'the site description for %s has no docker description or no container_name'
% opts.name)
return
# collect info on database container which allways is named 'db'
if not self.opts.docker_build_image:
self.update_docker_info(self.db_container_name, required=True)
self.update_docker_info(docker['container_name'])
def fetchImage(imageURL, forcepull=False):
pullImage = False
imageReady = False
image = None
try:
image = docker.get(imageURL)
imageReady = True
except docker.errors.ImageNotFound:
# need to pull image
pullImage = True
except docker.errors.APIError:
print("Docker API Error. Exiting without executing task!")
# should raise error
return None
if forcepull == True:
pullImage = True
if pullImage == True or imageReady == False:
image = docker.pull(imageURL)
if image == None:
## Maybe the image doesn't exist? Maybe docker.pull will throw and APIError?
print("Image not found. Returning without executing task!")
# should raise error
return None
return image
def run_docker(address, interval, host, port, debug=False):
prev_cpu, prev_system = {}, {}
prev_tx_bytes, prev_rx_bytes, prev_timer = {}, {}, {}
client = statsd.StatsClient(host, port)
MEM_USAGE = jmespath.compile('memory_stats.usage')
MEM_LIMIT = jmespath.compile('memory_stats.limit')
TOTAL_USAGE = jmespath.compile('cpu_stats.cpu_usage.total_usage')
SYSTEM_USAGE = jmespath.compile('cpu_stats.system_cpu_usage')
NUM_CPUS = jmespath.compile('length(cpu_stats.cpu_usage.percpu_usage)')
TX_BYTES = jmespath.compile('networks.eth0.tx_bytes') # TODO: Always eth0??? (likely not...)
RX_BYTES = jmespath.compile('networks.eth0.rx_bytes')
try:
while True:
with client.pipeline() as pipe:
start = time.time()
containers = get(address, '/containers/json?all=1', debug)
for container in containers:
name = container.get('Names')[0].strip('/')
status = container.get('Status')
id_ = container.get('Id')
log.debug("{}: {}".format(name, status))
stats = get(address, '/containers/{}/stats?stream=0'.format(id_), debug) # Very slow call...
mem_usage = MEM_USAGE.search(stats) or 0
mem_limit = MEM_LIMIT.search(stats) or 1
mem_percent = 100.0 * (mem_usage / mem_limit) if mem_limit > 0 else 0
if debug:
log.debug("{}: Mem: {:,} {:,} {}%".format(name, mem_usage, mem_limit, mem_percent))
pipe.gauge('system.memory.virtual.percent,service={}'.format(name), mem_percent)
# http://stackoverflow.com/questions/30271942/get-docker-container-cpu-usage-as-percentage
cpu_percent = 0
total_usage = TOTAL_USAGE.search(stats) or 0
cpu_delta = total_usage - prev_cpu.get(name, 0)
system_usage = SYSTEM_USAGE.search(stats) or 0
system_delta = system_usage - prev_system.get(name, 0)
num_cpus = NUM_CPUS.search(stats) or 1
if system_delta > 0 and cpu_delta > 0:
cpu_percent = (cpu_delta / system_delta) * num_cpus * 100.0
if debug:
log.debug("{}: Cpu: {}, {}: {}%".format(name, cpu_delta, system_delta, cpu_percent))
prev_cpu[name], prev_system[name] = total_usage, system_usage
pipe.gauge('system.cpu.percent,service={}'.format(name), cpu_percent)
tx_bytes = TX_BYTES.search(stats) or 0
rx_bytes = RX_BYTES.search(stats) or 0
tx = tx_bytes - prev_tx_bytes.setdefault(name, 0) # B
rx = rx_bytes - prev_rx_bytes.setdefault(name, 0)
timer = time.time()
elapsed = timer - prev_timer.get(name, 0) # s
prev_timer[name] = timer
tx_rate = tx / elapsed if tx > 0 and elapsed > 0 else 0 # B/s
rx_rate = rx / elapsed if rx > 0 and elapsed > 0 else 0
pipe.gauge('system.network.send_rate,service={}'.format(name), tx_rate)
pipe.gauge('system.network.recv_rate,service={}'.format(name), rx_rate)
if debug:
log.debug("{}: Net Tx: {:,} -> {:,} ({}B/s)".format(name, tx_bytes, prev_tx_bytes[name], tx_rate))
log.debug("{}: Net Rx: {:,} -> {:,} ({}B/s)".format(name, rx_bytes, prev_rx_bytes[name], rx_rate))
prev_tx_bytes[name] = tx_bytes
prev_rx_bytes[name] = rx_bytes
pipe.gauge('system.disk.root.percent,service={}'.format(name), 0)
elapsed = time.time() - start
log.debug("docker: {}ms".format(int(elapsed * 1000)))
time.sleep(interval - elapsed)
except Exception as e:
log.exception(e)
def run_docker(address, interval, host, port, debug=False):
prev_cpu, prev_system = {}, {}
prev_tx_bytes, prev_rx_bytes, prev_timer = {}, {}, {}
client = statsd.StatsClient(host, port)
MEM_USAGE = jmespath.compile('memory_stats.usage')
MEM_LIMIT = jmespath.compile('memory_stats.limit')
TOTAL_USAGE = jmespath.compile('cpu_stats.cpu_usage.total_usage')
SYSTEM_USAGE = jmespath.compile('cpu_stats.system_cpu_usage')
NUM_CPUS = jmespath.compile('length(cpu_stats.cpu_usage.percpu_usage)')
TX_BYTES = jmespath.compile(
'networks.eth0.tx_bytes') # TODO: Always eth0??? (likely not...)
RX_BYTES = jmespath.compile('networks.eth0.rx_bytes')
try:
while True:
with client.pipeline() as pipe:
start = time.time()
containers = get(address, '/containers/json?all=1', debug)
for container in containers:
name = container.get('Names')[0].strip('/')
status = container.get('Status')
id_ = container.get('Id')
log.debug("{}: {}".format(name, status))
stats = get(address,
'/containers/{}/stats?stream=0'.format(id_),
debug) # Very slow call...
mem_usage = MEM_USAGE.search(stats) or 0
mem_limit = MEM_LIMIT.search(stats) or 1
mem_percent = 100.0 * (mem_usage /
mem_limit) if mem_limit > 0 else 0
if debug:
log.debug("{}: Mem: {:,} {:,} {}%".format(
name, mem_usage, mem_limit, mem_percent))
pipe.gauge(
'system.memory.virtual.percent,service={}'.format(
name), mem_percent)
# http://stackoverflow.com/questions/30271942/get-docker-container-cpu-usage-as-percentage
cpu_percent = 0
total_usage = TOTAL_USAGE.search(stats) or 0
cpu_delta = total_usage - prev_cpu.get(name, 0)
system_usage = SYSTEM_USAGE.search(stats) or 0
system_delta = system_usage - prev_system.get(name, 0)
num_cpus = NUM_CPUS.search(stats) or 1
if system_delta > 0 and cpu_delta > 0:
cpu_percent = (cpu_delta /
system_delta) * num_cpus * 100.0
if debug:
log.debug("{}: Cpu: {}, {}: {}%".format(
name, cpu_delta, system_delta, cpu_percent))
prev_cpu[name], prev_system[
name] = total_usage, system_usage
pipe.gauge('system.cpu.percent,service={}'.format(name),
cpu_percent)
tx_bytes = TX_BYTES.search(stats) or 0
rx_bytes = RX_BYTES.search(stats) or 0
tx = tx_bytes - prev_tx_bytes.setdefault(name, 0) # B
rx = rx_bytes - prev_rx_bytes.setdefault(name, 0)
timer = time.time()
elapsed = timer - prev_timer.get(name, 0) # s
prev_timer[name] = timer
tx_rate = tx / elapsed if tx > 0 and elapsed > 0 else 0 # B/s
rx_rate = rx / elapsed if rx > 0 and elapsed > 0 else 0
pipe.gauge(
'system.network.send_rate,service={}'.format(name),
tx_rate)
pipe.gauge(
'system.network.recv_rate,service={}'.format(name),
rx_rate)
if debug:
log.debug("{}: Net Tx: {:,} -> {:,} ({}B/s)".format(
name, tx_bytes, prev_tx_bytes[name], tx_rate))
log.debug("{}: Net Rx: {:,} -> {:,} ({}B/s)".format(
name, rx_bytes, prev_rx_bytes[name], rx_rate))
prev_tx_bytes[name] = tx_bytes
prev_rx_bytes[name] = rx_bytes
pipe.gauge(
'system.disk.root.percent,service={}'.format(name), 0)
elapsed = time.time() - start
log.debug("docker: {}ms".format(int(elapsed * 1000)))
time.sleep(interval - elapsed)
except Exception as e:
log.exception(e)