def disconnect_instance(self, instance_num):
blueprint = self.blueprint
allocation = self.allocation
instance_id = self.group_id + '_' + instance_num
addr = blueprint['instances'][instance_num]['addr']
memsize = blueprint['memsize']
network_settings = Sense.network_settings()
network_name = network_settings['network_name']
if not network_name:
raise RuntimeError("Network name is not specified in settings")
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
try:
docker_obj.disconnect_container_from_network(instance_id,
network_name,
force=True)
except:
pass
def remove_container(self, instance_num):
containers = self.containers
if instance_num not in containers['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_hosts = [h['addr'].split(':')[0] for h in Sense.docker_hosts()
if h['status'] == 'passing']
if containers:
docker_host = containers['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
logging.info("Removing container '%s' from '%s'",
instance_id,
docker_host)
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
docker_obj.stop(container=instance_id)
docker_obj.remove_container(container=instance_id)
else:
logging.info("Not removing container '%s', as it doesn't exist",
instance_id)
def allocate_ip(skip=[]):
global IP_CACHE
global CACHE_LOCK
docker_nodes = [h for h in Sense.docker_hosts() if h['status'] == 'passing']
network_settings = Sense.network_settings()
subnet = network_settings['subnet']
gateway_ip = network_settings['gateway_ip']
if gateway_ip:
skip += [gateway_ip]
if not subnet:
raise RuntimeError("Subnet is not specified in settings")
invalidate_cache()
with CACHE_LOCK:
allocated_ips = set(IP_CACHE.keys())
# collect instances from blueprints
for blueprint in Sense.blueprints().values():
for instance in blueprint['instances'].values():
allocated_ips.add(instance['addr'])
net = ipaddress.ip_network(subnet)
except_list = allocated_ips.union(set(skip))
for addr in net:
if str(addr) not in except_list and\
not str(addr).endswith('.0'):
IP_CACHE[str(addr)] = datetime.datetime.now()
return str(addr)
raise RuntimeError('IP Address range exhausted')
def delete(self, delete_task):
try:
group_id = self.group_id
delete_task.log("Unallocating instance")
self.unallocate()
delete_task.log("Unregistering services")
self.unregister()
delete_task.log("Removing containers")
self.remove_containers()
delete_task.log("Removing blueprint")
self.remove_blueprint()
delete_task.log("Completed removing group")
Sense.update()
delete_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to delete group '%s'", group_id)
delete_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def delete(self, delete_task):
try:
group_id = self.group_id
delete_task.log("Unallocating instance")
self.unallocate()
delete_task.log("Unregistering services")
self.unregister()
delete_task.log("Removing containers")
self.remove_containers()
delete_task.log("Removing blueprint")
self.remove_blueprint()
delete_task.log("Completed removing group")
Sense.update()
delete_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to delete group '%s'", group_id)
delete_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def update(self, name, memsize, password,
docker_image_name, heal, backup_id, storage, update_task):
try:
if heal:
self.heal(update_task)
if name and name != self.blueprint['name']:
self.rename(name, update_task)
if memsize and memsize != self.blueprint['memsize']:
self.resize(memsize, update_task)
if password:
self.set_password(password, update_task)
if docker_image_name:
self.upgrade(update_task)
if backup_id:
self.restore(backup_id, storage, update_task)
Sense.update()
update_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to update group '%s'", self.group_id)
update_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def allocate(memory, anti_affinity = []):
docker_hosts = [h for h in Sense.docker_hosts()
if (h['status'] == 'passing' and
'im' in h['tags'])]
if not docker_hosts:
raise RuntimeError("There are no healthy docker nodes")
blueprints = Sense.blueprints()
allocations = Sense.allocations()
memory_used = {h['addr'].split(':')[0]: 0 for h in docker_hosts}
for group_id, blueprint in blueprints.items():
if group_id not in allocations:
continue
memsize = blueprint['memsize']
for instance in allocations[group_id]['instances'].values():
host = instance['host'].split(':')[0]
memory_used[host] = memory_used.get(host, 0) + memsize
scores = []
for docker_host in docker_hosts:
addr = docker_host['addr'].split(':')[0]
free_mem = docker_host['memory'] - memory_used[addr]
affinity = 0 if addr in anti_affinity else 1
scores.append((affinity, free_mem, docker_host))
sorted_scores = sorted(scores, reverse=True,
key=lambda k: k[0:2])
for score in sorted_scores:
docker_host = score[2]
addr = docker_host['addr'].split(':')[0]
free_mem = docker_host['memory'] - memory_used[addr]
if free_mem > memory:
logging.info("Allocating new instance with %d MiB memory at '%s'",
memory,
addr)
return addr
docker_host = sorted_scores[0][2]
addr = docker_host['addr'].split(':')[0]
logging.info("There were no hosts with %d MiB of free memory, " +
"so allocating instance on '%s'",
memory,
addr)
return addr
def allocate(memory, anti_affinity=[]):
docker_hosts = [
h for h in Sense.docker_hosts()
if (h['status'] == 'passing' and 'im' in h['tags'])
]
if not docker_hosts:
raise RuntimeError("There are no healthy docker nodes")
blueprints = Sense.blueprints()
allocations = Sense.allocations()
memory_used = {h['addr'].split(':')[0]: 0 for h in docker_hosts}
for group_id, blueprint in blueprints.items():
if group_id not in allocations:
continue
memsize = blueprint['memsize']
for instance in allocations[group_id]['instances'].values():
host = instance['host'].split(':')[0]
memory_used[host] = memory_used.get(host, 0) + memsize
scores = []
for docker_host in docker_hosts:
addr = docker_host['addr'].split(':')[0]
free_mem = docker_host['memory'] - memory_used[addr]
affinity = 0 if addr in anti_affinity else 1
scores.append((affinity, free_mem, docker_host))
sorted_scores = sorted(scores, reverse=True, key=lambda k: k[0:2])
for score in sorted_scores:
docker_host = score[2]
addr = docker_host['addr'].split(':')[0]
free_mem = docker_host['memory'] - memory_used[addr]
if free_mem > memory:
logging.info("Allocating new instance with %d MiB memory at '%s'",
memory, addr)
return addr
docker_host = sorted_scores[0][2]
addr = docker_host['addr'].split(':')[0]
logging.info(
"There were no hosts with %d MiB of free memory, " +
"so allocating instance on '%s'", memory, addr)
return addr
def __init__(self):
self.nlp = spacy.load('en')
self.word_vectors = sense2vec.load()
self.handlers = {
'similar': Sense(self.nlp, self.word_vectors),
'parse': Parse(self.nlp),
'vector': Sense(self.nlp, self.word_vectors),
#'intent': Intent(self.nlp, self.word_vectors)
# 'converse':
# 'person':
# 'address':
# 'date':
# 'email':
}
def __init__(self):
config = configparser.ConfigParser()
config.read("config.ini")
IN_1 = config.getint("car","IN1")
IN_2 = config.getint("car","IN2")
IN_3 = config.getint("car","IN3")
IN_4 = config.getint("car","IN4")
self.change_speed = ChangeSpeed()
self.motor = Motor(IN_1, IN_2, IN_3, IN_4)
self.sense = Sense()
self.qaudio = QAudio(4000)
self.led_light = LED_Light()
def containers(self):
containers = Sense.containers()
if self.group_id in containers:
return containers[self.group_id]
else:
return {"instances": {}}
def ensure_network(self, docker_addr):
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
settings = Sense.network_settings()
network_name = settings['network_name']
subnet = settings['subnet']
if not network_name:
raise RuntimeError("Network name not specified")
network_exists = any([n['Name'] == network_name
for n in docker_obj.networks()])
if network_exists:
return
if not settings['create_automatically']:
raise RuntimeError(("No network '%s' exists and automatic creation" +
"prohibited") % network_name)
ipam_pool = docker.utils.create_ipam_pool(
subnet=subnet
)
ipam_config = docker.utils.create_ipam_config(
pool_configs=[ipam_pool]
)
logging.info("Creating network '%s'", network_name)
docker_obj.create_network(name=network_name,
driver='bridge',
ipam=ipam_config)
def ensure_network(self, docker_addr):
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
settings = Sense.network_settings()
network_name = settings['network_name']
subnet = settings['subnet']
if not network_name:
raise RuntimeError("Network name not specified")
network_exists = any([n['Name'] == network_name
for n in docker_obj.networks()])
if network_exists:
return
if not settings['create_automatically']:
raise RuntimeError(("No network '%s' exists and automatic creation" +
"prohibited") % network_name)
ipam_pool = docker.utils.create_ipam_pool(
subnet=subnet
)
ipam_config = docker.utils.create_ipam_config(
pool_configs=[ipam_pool]
)
logging.info("Creating network '%s'", network_name)
docker_obj.create_network(name=network_name,
driver='bridge',
ipam=ipam_config)
def services(self):
services = Sense.services()
if self.group_id in services:
return services[self.group_id]
else:
return {"instances": {}}
def allocation(self):
allocations = Sense.allocations()
if self.group_id in allocations:
return allocations[self.group_id]
else:
return {"instances": {}}
def blueprint(self):
blueprints = Sense.blueprints()
if self.group_id in blueprints:
self._blueprint = blueprints[self.group_id]
return self._blueprint
def create(cls, create_task, name, memsize, password, check_period):
group_id = create_task.group_id
try:
consul_obj = consul.Consul(host=global_env.consul_host,
token=global_env.consul_acl_token)
kv = consul_obj.kv
create_task.log("Creating group '%s'", group_id)
ip1 = ip_pool.allocate_ip()
ip2 = ip_pool.allocate_ip()
creation_time = datetime.datetime.now(
datetime.timezone.utc).isoformat()
kv.put('tarantool/%s/blueprint/type' % group_id, 'memcached')
kv.put('tarantool/%s/blueprint/name' % group_id,
name.encode('utf-8'))
kv.put('tarantool/%s/blueprint/memsize' % group_id, str(memsize))
kv.put('tarantool/%s/blueprint/check_period' % group_id,
str(check_period))
kv.put('tarantool/%s/blueprint/creation_time' % group_id,
creation_time)
kv.put('tarantool/%s/blueprint/instances/1/addr' % group_id, ip1)
kv.put('tarantool/%s/blueprint/instances/2/addr' % group_id, ip2)
Sense.update()
memc = Memcached(global_env.consul_host, group_id)
create_task.log("Allocating instance to physical nodes")
memc.allocate()
Sense.update()
create_task.log("Registering services")
memc.register()
Sense.update()
create_task.log("Creating containers")
memc.create_containers(password)
Sense.update()
create_task.log("Enabling replication")
memc.wait_for_instances(create_task)
memc.enable_replication()
create_task.log("Completed creating group")
create_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to create group '%s'", group_id)
create_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
return memc
def resize_instance(self, instance_num, memsize):
containers = self.containers
if instance_num not in containers['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_hosts = [h['addr'].split(':')[0] for h in Sense.docker_hosts()
if h['status'] == 'passing']
if containers:
docker_host = containers['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
logging.info("Resizing container '%s' to %d MiB on '%s'",
instance_id,
memsize,
docker_host)
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
cmd = "tarantool_set_config.lua TARANTOOL_SLAB_ALLOC_ARENA " + \
str(float(memsize)/1024)
exec_id = docker_obj.exec_create(self.group_id + '_' + instance_num,
cmd)
docker_obj.exec_start(exec_id)
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] != 0:
raise RuntimeError("Failed to set memory size for container " +
instance_id)
docker_obj.restart(container=instance_id)
else:
logging.info("Not resizing container '%s', as it doesn't exist",
instance_id)
def resize_instance(self, instance_num, memsize):
containers = self.containers
if instance_num not in containers['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_hosts = [h['addr'].split(':')[0] for h in Sense.docker_hosts()
if h['status'] == 'passing']
if containers:
docker_host = containers['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
logging.info("Resizing container '%s' to %d MiB on '%s'",
instance_id,
memsize,
docker_host)
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
cmd = "tarantool_set_config.lua TARANTOOL_SLAB_ALLOC_ARENA " + \
str(float(memsize)/1024)
exec_id = docker_obj.exec_create(self.group_id + '_' + instance_num,
cmd)
docker_obj.exec_start(exec_id)
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] != 0:
raise RuntimeError("Failed to set memory size for container " +
instance_id)
docker_obj.restart(container=instance_id)
else:
logging.info("Not resizing container '%s', as it doesn't exist",
instance_id)
def get_instance_password(self, instance_num):
containers = self.containers
if instance_num not in containers['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_hosts = [
h['addr'].split(':')[0] for h in Sense.docker_hosts()
if h['status'] == 'passing'
]
if containers:
docker_host = containers['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
logging.info("Getting password for '%s' on '%s'", instance_id,
docker_host)
docker_obj = docker.APIClient(base_url=docker_addr,
tls=global_env.docker_tls_config)
try:
strm, stat = docker_obj.get_archive(
instance_id, '/opt/tarantool/auth.sasldb')
bio = io.BytesIO()
shutil.copyfileobj(strm, bio)
bio.seek(0)
tar = tarfile.open(fileobj=bio)
fobj = tar.extractfile('auth.sasldb')
return base64.b64encode(gzip.compress(fobj.read()))
except docker.errors.NotFound:
return None
else:
raise RuntimeError("No such container: %s", instance_id)
def register(self):
instance_num = '1'
blueprint = self.blueprint
allocation = self.allocation
instance_id = self.group_id + '_' + instance_num
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
consul_host = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
consul_host = host['consul_host']
if not consul_host:
raise RuntimeError("Failed to find consul host of %s" %
docker_host)
addr = blueprint['instances'][instance_num]['addr']
check_period = blueprint['check_period']
consul_obj = consul.Consul(host=consul_host,
token=global_env.consul_acl_token)
container_check = {
'docker_container_id': instance_id,
'shell': "/bin/sh",
'script': "/bin/true",
'interval': "%ds" % check_period,
'status': 'warning'
}
replication_check = {
'docker_container_id': instance_id,
'shell': "/bin/sh",
'script': "/var/lib/mon.d/tarantool_replication.sh",
'interval': "%ds" % check_period,
'status': 'warning'
}
memory_check = {
'docker_container_id': instance_id,
'shell': "/bin/sh",
'script': "/var/lib/mon.d/tarantool_memory.sh",
'interval': "%ds" % check_period,
'status': 'warning'
}
logging.info("Registering instance '%s' on '%s'",
instance_id,
consul_host)
ret = consul_obj.agent.service.register("tarantino",
service_id=instance_id,
address=addr,
port=80,
check=container_check,
tags=['tarantool'])
def set_instance_password(self, instance_num, password):
containers = self.containers
if instance_num not in containers['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_hosts = [
h['addr'].split(':')[0] for h in Sense.docker_hosts()
if h['status'] == 'passing'
]
if containers:
docker_host = containers['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
logging.info("Setting password for '%s' on '%s'", instance_id,
docker_host)
docker_obj = docker.APIClient(base_url=docker_addr,
tls=global_env.docker_tls_config)
cmd = "memcached_set_password.lua " + password
exec_id = docker_obj.exec_create(
self.group_id + '_' + instance_num, cmd)
docker_obj.exec_start(exec_id)
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] != 0:
raise RuntimeError("Failed to set password for container " +
instance_id)
else:
logging.info("Not setting password for '%s', as it doesn't exist",
instance_id)
def register(self):
instance_num = '1'
blueprint = self.blueprint
allocation = self.allocation
instance_id = self.group_id + '_' + instance_num
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
consul_host = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
consul_host = host['consul_host']
if not consul_host:
raise RuntimeError("Failed to find consul host of %s" %
docker_host)
addr = blueprint['instances'][instance_num]['addr']
check_period = blueprint['check_period']
consul_obj = consul.Consul(host=consul_host,
token=global_env.consul_acl_token)
container_check = {
'docker_container_id': instance_id,
'shell': "/bin/sh",
'script': "/bin/true",
'interval': "%ds" % check_period,
'status': 'warning'
}
replication_check = {
'docker_container_id': instance_id,
'shell': "/bin/sh",
'script': "/var/lib/mon.d/tarantool_replication.sh",
'interval': "%ds" % check_period,
'status': 'warning'
}
memory_check = {
'docker_container_id': instance_id,
'shell': "/bin/sh",
'script': "/var/lib/mon.d/tarantool_memory.sh",
'interval': "%ds" % check_period,
'status': 'warning'
}
logging.info("Registering instance '%s' on '%s'", instance_id,
consul_host)
ret = consul_obj.agent.service.register("tarantino",
service_id=instance_id,
address=addr,
port=80,
check=container_check,
tags=['tarantool'])
def upgrade(self, upgrade_task):
try:
group_id = self.group_id
upgrade_task.log("Upgrading container 1")
self.upgrade_container("1")
upgrade_task.log("Upgrading container 2")
self.upgrade_container("2")
upgrade_task.log("Completed upgrading containers")
Sense.update()
upgrade_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to upgrade group '%s'", group_id)
upgrade_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def update_config(self, instance_num, config_str):
containers = self.containers
if instance_num not in containers['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_hosts = [h['addr'].split(':')[0] for h in Sense.docker_hosts()
if h['status'] == 'passing']
if containers:
docker_host = containers['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
logging.info("Uploading new config for container '%s' on '%s'",
instance_id,
docker_host)
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
buf = io.BytesIO(tar_string('service.json', config_str))
status = docker_obj.put_archive(self.group_id + '_' + instance_num,
'/opt/tarantool',
buf)
if not status:
raise RuntimeError("Failed to set config for container " +
instance_id)
docker_obj.restart(container=instance_id)
else:
logging.info(
"Not setting config for container '%s', as it doesn't exist",
instance_id)
def update_config(self, instance_num, config_str):
containers = self.containers
if instance_num not in containers['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_hosts = [
h['addr'].split(':')[0] for h in Sense.docker_hosts()
if h['status'] == 'passing'
]
if containers:
docker_host = containers['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
logging.info("Uploading new config for container '%s' on '%s'",
instance_id, docker_host)
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
buf = io.BytesIO(tar_string('service.json', config_str))
status = docker_obj.put_archive(self.group_id + '_' + instance_num,
'/opt/tarantool', buf)
if not status:
raise RuntimeError("Failed to set config for container " +
instance_id)
docker_obj.restart(container=instance_id)
else:
logging.info(
"Not setting config for container '%s', as it doesn't exist",
instance_id)
def unregister_instance(self, instance_num):
services = self.services
allocation = self.allocation
if instance_num not in services['instances']:
return
instance_id = self.group_id + '_' + instance_num
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
consul_host = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
consul_host = host['consul_host']
if not consul_host:
raise RuntimeError("Failed to find consul host of %s" %
docker_host)
consul_hosts = [
h['addr'].split(':')[0] for h in Sense.consul_hosts()
if h['status'] == 'passing'
]
if services:
if consul_host in consul_hosts:
consul_obj = consul.Consul(host=consul_host,
token=global_env.consul_acl_token)
check_id = instance_id + '_memory'
logging.info("Unregistering check '%s'", check_id)
consul_obj.agent.check.deregister(check_id)
consul_obj.agent.check.deregister('service:' + instance_id)
logging.info("Unregistering instance '%s' from '%s'",
instance_id, consul_host)
consul_obj.agent.service.deregister(instance_id)
else:
logging.info("Not unregistering '%s', as it's not registered",
instance_id)
def _buildSenseAct(self, agent, sense_name):
"""Returns a sense-act object for the given name.
@param agent: The agent to build the sense-act for.
@type agent: L{POSH.strict.Agent}
@param sense_name: The name of the sense-act.
@type sense_name: string
@return: The created sense-act object
@rtype: L{POSH.strict.Sense}
"""
return Sense(agent, sense_name)
def __init__(self, consul_host, group_id):
self.consul_host = consul_host
self.consul = consul.Consul(host=consul_host,
token=global_env.consul_acl_token)
self.group_id = group_id
blueprints = Sense.blueprints()
if not group_id in blueprints:
raise GroupNotFoundError("No such blueprint: '%s'", group_id)
self._blueprint = blueprints[group_id]
def proc_elem(this_elem):
if type(this_elem) is ListType:
return map(proc_elem, this_elem)
elif (type(this_elem) is InstanceType) and \
isinstance(this_elem, Sense):
return Sense(sensor=this_elem.sensor,
sense_value=sense_value,
sense_predicate=sense_predicate,
drive_name=drive_name,
agent=self.agent)
else:
return this_elem
def create(cls, create_task, name, memsize, password, check_period):
group_id = create_task.group_id
try:
consul_obj = consul.Consul(host=global_env.consul_host,
token=global_env.consul_acl_token)
kv = consul_obj.kv
create_task.log("Creating group '%s'", group_id)
ip1 = ip_pool.allocate_ip()
creation_time = datetime.datetime.now(
datetime.timezone.utc).isoformat()
kv.put('tarantool/%s/blueprint/type' % group_id, 'tarantino')
kv.put('tarantool/%s/blueprint/name' % group_id, name)
kv.put('tarantool/%s/blueprint/memsize' % group_id, str(memsize))
kv.put('tarantool/%s/blueprint/check_period' % group_id,
str(check_period))
kv.put('tarantool/%s/blueprint/creation_time' % group_id,
creation_time)
kv.put('tarantool/%s/blueprint/instances/1/addr' % group_id, ip1)
Sense.update()
tar = Tarantino(global_env.consul_host, group_id)
create_task.log("Allocating instance to physical nodes")
tar.allocate()
Sense.update()
create_task.log("Registering services")
tar.register()
Sense.update()
create_task.log("Creating containers")
tar.create_containers(password)
Sense.update()
create_task.log("Completed creating group")
create_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to create group '%s'", group_id)
create_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
return tar
def enable_replication(self):
port = 3301
blueprint = self.blueprint
allocation = self.allocation
for instance_num in allocation['instances']:
other_instances = \
set(allocation['instances'].keys()) - set([instance_num])
addr = blueprint['instances'][instance_num]['addr']
other_addrs = [blueprint['instances'][i]['addr']
for i in other_instances]
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
logging.info("Enabling replication between '%s' and '%s'",
addr, str(other_addrs))
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
cmd = "tarantool_set_config.lua TARANTOOL_REPLICATION_SOURCE " + \
",".join(other_addrs)
attempts = 0
while attempts < 5:
exec_id = docker_obj.exec_create(self.group_id + '_' + instance_num,
cmd)
stream = docker_obj.exec_start(exec_id, stream=True)
for line in stream:
logging.info("Exec: %s", str(line))
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] == 0:
break
time.sleep(1)
attempts+=1
if attempts >= 5:
raise RuntimeError("Failed to enable replication for group " +
self.group_id)
def update(self, name, memsize, password, config_str, docker_image_name,
update_task):
try:
if name and name != self.blueprint['name']:
self.rename(name, update_task)
if memsize and memsize != self.blueprint['memsize']:
self.resize(memsize, update_task)
if config_str:
update_task.log("Updating config of group %s", self.group_id)
self.update_config("1", config_str)
if docker_image_name:
self.upgrade(update_task)
Sense.update()
update_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to update group '%s'", self.group_id)
update_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def update(self, name, memsize, password, config_str,
docker_image_name, update_task):
try:
if name and name != self.blueprint['name']:
self.rename(name, update_task)
if memsize and memsize != self.blueprint['memsize']:
self.resize(memsize, update_task)
if config_str:
update_task.log("Updating config of group %s", self.group_id)
self.update_config("1", config_str)
if docker_image_name:
self.upgrade(update_task)
Sense.update()
update_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to update group '%s'", self.group_id)
update_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def wait_for_instances(self, wait_task):
port = 3301
blueprint = self.blueprint
allocation = self.allocation
for instance_num in allocation['instances']:
other_instances = \
set(allocation['instances'].keys()) - set([instance_num])
addr = blueprint['instances'][instance_num]['addr']
other_addrs = [
blueprint['instances'][i]['addr'] for i in other_instances
]
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
instance_id = self.group_id + '_' + instance_num
wait_task.log(
"Waiting for '%s' to go up. It may take time to " +
"load data from disk.", instance_id)
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
docker_obj = docker.APIClient(base_url=docker_addr,
tls=global_env.docker_tls_config)
cmd = "tarantool_is_up"
attempts = 0
while True:
exec_id = docker_obj.exec_create(instance_id, cmd)
stream = docker_obj.exec_start(exec_id, stream=True)
for line in stream:
logging.info("Exec: %s", str(line))
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] == 0:
break
wait_task.log("Waiting for '%s' to go up. Attempt %d.",
instance_id, attempts)
time.sleep(1)
attempts += 1
def _buildSense(self, agent, sense_struct):
"""Returns a sense object for the given structure.
The structure is of the form (name, value, predicate), where
the first is a string, and the other two elements are either
a string or None.
@param agent: The agent to build the sense for.
@type agent: L{POSH.strict.Agent}
@param sense_struct: The sense structure.
@type sense_struct: described above
@return: The created sense object
@rtype: L{POSH.strict.Sense}
@raise NameError: If the sense could not be found in the
behaviour dictionary.
"""
return Sense(agent, sense_struct[0], sense_struct[1], sense_struct[2])
from sense import Sense sensors = Sense() log = sensors.read_log() # print log conditions = sensors.current_conditions() print 'Last sensor log entry:', conditions[0] print 'Temperature in deg C:', conditions[1] print 'Pressure in mBar:', conditions[2] print 'Relative Humidity:', conditions[3] print 'Last reading was taken at:', conditions[4]
def upgrade_container(self, instance_num):
group_id = self.group_id
logging.info("Upgrading container '%s'", group_id)
blueprint = self.blueprint
allocation = self.allocation
instance_id = self.group_id + '_' + instance_num
addr = blueprint['instances'][instance_num]['addr']
memsize = blueprint['memsize']
network_settings = Sense.network_settings()
network_name = network_settings['network_name']
if not network_name:
raise RuntimeError("Network name is not specified in settings")
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
replica_ip = None
if instance_num == '2':
replica_ip = blueprint['instances']['1']['addr']
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
self.ensure_image(docker_addr)
self.ensure_network(docker_addr)
mounts = docker_obj.inspect_container(instance_id)["Mounts"]
binds = []
for mount in mounts:
if mount['Destination'] == '/opt/tarantool':
# code should be upgraded along with container
continue
logging.info("Keeping mount %s:%s",
mount["Source"], mount["Destination"])
rw_flag = "rw" if mount['RW'] else "ro"
binds.append("%s:%s:%s" % (mount['Source'],
mount['Destination'],
rw_flag))
docker_obj.stop(container=instance_id)
docker_obj.remove_container(container=instance_id)
host_config = docker_obj.create_host_config(
restart_policy =
{
"MaximumRetryCount": 0,
"Name": "unless-stopped"
},
binds = binds
)
cmd = 'tarantool /opt/tarantool/app.lua'
networking_config = {
'EndpointsConfig':
{
network_name:
{
'IPAMConfig':
{
"IPv4Address": addr,
"IPv6Address": ""
},
"Links": [],
"Aliases": []
}
}
}
environment = {}
environment['TARANTOOL_SLAB_ALLOC_ARENA'] = float(memsize)/1024
if replica_ip:
environment['TARANTOOL_REPLICATION_SOURCE'] = replica_ip + ':3301'
container = docker_obj.create_container(image='tarantool-cloud-memcached',
name=instance_id,
command=cmd,
host_config=host_config,
networking_config=networking_config,
environment=environment,
labels=['tarantool'])
docker_obj.connect_container_to_network(container.get('Id'),
network_name,
ipv4_address=addr)
docker_obj.start(container=container.get('Id'))
def create_container(self, instance_num,
other_instance_num,
password,
password_base64):
blueprint = self.blueprint
allocation = self.allocation
instance_id = self.group_id + '_' + instance_num
addr = blueprint['instances'][instance_num]['addr']
memsize = blueprint['memsize']
network_settings = Sense.network_settings()
network_name = network_settings['network_name']
if not network_name:
raise RuntimeError("Network name is not specified in settings")
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
replica_ip = None
if other_instance_num is not None:
replica_ip = blueprint['instances'][other_instance_num]['addr']
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
self.ensure_image(docker_addr)
self.ensure_network(docker_addr)
if not replica_ip:
logging.info("Creating memcached '%s' on '%s' with ip '%s'",
instance_id, docker_obj.base_url, addr)
else:
logging.info("Creating memcached '%s' on '%s' with ip '%s'" +
" and replication source: '%s'",
instance_id, docker_obj.base_url, addr, replica_ip)
host_config = docker_obj.create_host_config(
restart_policy =
{
"MaximumRetryCount": 0,
"Name": "unless-stopped"
})
cmd = 'tarantool /opt/tarantool/app.lua'
networking_config = {
'EndpointsConfig':
{
network_name:
{
'IPAMConfig':
{
"IPv4Address": addr,
"IPv6Address": ""
},
"Links": [],
"Aliases": []
}
}
}
environment = {}
environment['TARANTOOL_SLAB_ALLOC_ARENA'] = float(memsize)/1024
if password:
environment['MEMCACHED_PASSWORD'] = password
if password_base64:
environment['MEMCACHED_PASSWORD_BASE64'] = password_base64
if replica_ip:
environment['TARANTOOL_REPLICATION_SOURCE'] = replica_ip + ':3301'
container = docker_obj.create_container(image='tarantool-cloud-memcached',
name=instance_id,
command=cmd,
host_config=host_config,
networking_config=networking_config,
environment=environment,
labels=['tarantool'])
docker_obj.connect_container_to_network(container.get('Id'),
network_name,
ipv4_address=addr)
docker_obj.start(container=container.get('Id'))
def create_containers(self, password):
instance_num = '1'
blueprint = self.blueprint
allocation = self.allocation
instance_id = self.group_id + '_' + instance_num
addr = blueprint['instances'][instance_num]['addr']
memsize = blueprint['memsize']
network_settings = Sense.network_settings()
network_name = network_settings['network_name']
if not network_name:
raise RuntimeError("Network name is not specified in settings")
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
replica_ip = None
if instance_num == '2':
replica_ip = blueprint['instances']['1']['addr']
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
self.ensure_image(docker_addr)
self.ensure_network(docker_addr)
if not replica_ip:
logging.info("Creating tarantino '%s' on '%s' with ip '%s'",
instance_id, docker_obj.base_url, addr)
else:
logging.info("Creating tarantino '%s' on '%s' with ip '%s'" +
" and replication source: '%s'",
instance_id, docker_obj.base_url, addr, replica_ip)
host_config = docker_obj.create_host_config(
restart_policy =
{
"MaximumRetryCount": 0,
"Name": "unless-stopped"
})
networking_config = {
'EndpointsConfig':
{
network_name:
{
'IPAMConfig':
{
"IPv4Address": addr,
"IPv6Address": ""
},
"Links": [],
"Aliases": []
}
}
}
environment = {}
environment['TARANTOOL_SLAB_ALLOC_ARENA'] = float(memsize)/1024
if password:
environment['MEMCACHED_PASSWORD'] = password
container = docker_obj.create_container(image='tarantool/tarantino',
name=instance_id,
host_config=host_config,
networking_config=networking_config,
environment=environment,
labels=['tarantool'])
docker_obj.connect_container_to_network(container.get('Id'),
network_name,
ipv4_address=addr)
docker_obj.start(container=container.get('Id'))
