def update(self, name, memsize, password,
docker_image_name, heal, backup_id, storage, update_task):
try:
if heal:
self.heal(update_task)
if name and name != self.blueprint['name']:
self.rename(name, update_task)
if memsize and memsize != self.blueprint['memsize']:
self.resize(memsize, update_task)
if password:
self.set_password(password, update_task)
if docker_image_name:
self.upgrade(update_task)
if backup_id:
self.restore(backup_id, storage, update_task)
Sense.update()
update_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to update group '%s'", self.group_id)
update_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def delete(self, delete_task):
try:
group_id = self.group_id
delete_task.log("Unallocating instance")
self.unallocate()
delete_task.log("Unregistering services")
self.unregister()
delete_task.log("Removing containers")
self.remove_containers()
delete_task.log("Removing blueprint")
self.remove_blueprint()
delete_task.log("Completed removing group")
Sense.update()
delete_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to delete group '%s'", group_id)
delete_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def delete(self, delete_task):
try:
group_id = self.group_id
delete_task.log("Unallocating instance")
self.unallocate()
delete_task.log("Unregistering services")
self.unregister()
delete_task.log("Removing containers")
self.remove_containers()
delete_task.log("Removing blueprint")
self.remove_blueprint()
delete_task.log("Completed removing group")
Sense.update()
delete_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to delete group '%s'", group_id)
delete_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def create(cls, create_task, name, memsize, password, check_period):
group_id = create_task.group_id
try:
consul_obj = consul.Consul(host=global_env.consul_host,
token=global_env.consul_acl_token)
kv = consul_obj.kv
create_task.log("Creating group '%s'", group_id)
ip1 = ip_pool.allocate_ip()
ip2 = ip_pool.allocate_ip()
creation_time = datetime.datetime.now(
datetime.timezone.utc).isoformat()
kv.put('tarantool/%s/blueprint/type' % group_id, 'memcached')
kv.put('tarantool/%s/blueprint/name' % group_id,
name.encode('utf-8'))
kv.put('tarantool/%s/blueprint/memsize' % group_id, str(memsize))
kv.put('tarantool/%s/blueprint/check_period' % group_id,
str(check_period))
kv.put('tarantool/%s/blueprint/creation_time' % group_id,
creation_time)
kv.put('tarantool/%s/blueprint/instances/1/addr' % group_id, ip1)
kv.put('tarantool/%s/blueprint/instances/2/addr' % group_id, ip2)
Sense.update()
memc = Memcached(global_env.consul_host, group_id)
create_task.log("Allocating instance to physical nodes")
memc.allocate()
Sense.update()
create_task.log("Registering services")
memc.register()
Sense.update()
create_task.log("Creating containers")
memc.create_containers(password)
Sense.update()
create_task.log("Enabling replication")
memc.wait_for_instances(create_task)
memc.enable_replication()
create_task.log("Completed creating group")
create_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to create group '%s'", group_id)
create_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
return memc
def upgrade(self, upgrade_task):
try:
group_id = self.group_id
upgrade_task.log("Upgrading container 1")
self.upgrade_container("1")
upgrade_task.log("Upgrading container 2")
self.upgrade_container("2")
upgrade_task.log("Completed upgrading containers")
Sense.update()
upgrade_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to upgrade group '%s'", group_id)
upgrade_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def create(cls, create_task, name, memsize, password, check_period):
group_id = create_task.group_id
try:
consul_obj = consul.Consul(host=global_env.consul_host,
token=global_env.consul_acl_token)
kv = consul_obj.kv
create_task.log("Creating group '%s'", group_id)
ip1 = ip_pool.allocate_ip()
creation_time = datetime.datetime.now(
datetime.timezone.utc).isoformat()
kv.put('tarantool/%s/blueprint/type' % group_id, 'tarantino')
kv.put('tarantool/%s/blueprint/name' % group_id, name)
kv.put('tarantool/%s/blueprint/memsize' % group_id, str(memsize))
kv.put('tarantool/%s/blueprint/check_period' % group_id,
str(check_period))
kv.put('tarantool/%s/blueprint/creation_time' % group_id,
creation_time)
kv.put('tarantool/%s/blueprint/instances/1/addr' % group_id, ip1)
Sense.update()
tar = Tarantino(global_env.consul_host, group_id)
create_task.log("Allocating instance to physical nodes")
tar.allocate()
Sense.update()
create_task.log("Registering services")
tar.register()
Sense.update()
create_task.log("Creating containers")
tar.create_containers(password)
Sense.update()
create_task.log("Completed creating group")
create_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to create group '%s'", group_id)
create_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
return tar
def update(self, name, memsize, password, config_str,
docker_image_name, update_task):
try:
if name and name != self.blueprint['name']:
self.rename(name, update_task)
if memsize and memsize != self.blueprint['memsize']:
self.resize(memsize, update_task)
if config_str:
update_task.log("Updating config of group %s", self.group_id)
self.update_config("1", config_str)
if docker_image_name:
self.upgrade(update_task)
Sense.update()
update_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to update group '%s'", self.group_id)
update_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def update(self, name, memsize, password, config_str, docker_image_name,
update_task):
try:
if name and name != self.blueprint['name']:
self.rename(name, update_task)
if memsize and memsize != self.blueprint['memsize']:
self.resize(memsize, update_task)
if config_str:
update_task.log("Updating config of group %s", self.group_id)
self.update_config("1", config_str)
if docker_image_name:
self.upgrade(update_task)
Sense.update()
update_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to update group '%s'", self.group_id)
update_task.set_status(task.STATUS_CRITICAL, str(ex))
raise
def backup(self, backup_task, storage):
try:
services = self.services
backup_id = backup_task.backup_id
group_id = self.group_id
backup_task.log("Backing up group '%s'", group_id)
instance_num = '1'
allocation = self.allocation
instance_id = self.group_id + '_' + instance_num
docker_host = allocation['instances'][instance_num]['host']
docker_hosts = Sense.docker_hosts()
docker_addr = None
for host in docker_hosts:
if host['addr'].split(':')[0] == docker_host or \
host['consul_host'] == docker_host:
docker_addr = host['addr']
if not docker_addr:
raise RuntimeError("No such Docker host: '%s'" % docker_host)
docker_obj = docker.Client(base_url=docker_addr,
tls=global_env.docker_tls_config)
cmd = 'ls /var/lib/tarantool'
exec_id = docker_obj.exec_create(self.group_id + '_' + instance_num,
cmd)
out = docker_obj.exec_start(exec_id)
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] != 0:
raise RuntimeError("Failed to list snapshots for container " +
instance_id)
files = out.decode('utf-8').split('\n')
snapshots = [f for f in files if f.endswith('.snap')]
snapshot_lsns = sorted([os.path.splitext(s)[0] for s in snapshots])
xlogs = [f for f in files if f.endswith('.xlog')]
xlog_lsns = sorted([os.path.splitext(s)[0] for s in xlogs])
if not snapshot_lsns:
raise RuntimeError("There are no snapshots to backup")
latest_snapshot_lsn = snapshot_lsns[-1]
older_xlogs = list(filter(
lambda x: x <= latest_snapshot_lsn, xlog_lsns))
older_xlog = older_xlogs[-1]
newer_xlogs = list(filter(
lambda x: x > latest_snapshot_lsn, xlog_lsns))
xlogs_to_backup = [older_xlog] + newer_xlogs
files_to_backup = [latest_snapshot_lsn + '.snap']
files_to_backup += [xlog + '.xlog' for xlog in xlogs_to_backup]
backup_task.log("Backing up data: %s", ', '.join(files_to_backup))
tmp_backup_dir = '/var/lib/tarantool/backup-' + uuid.uuid4().hex
cmd = "mkdir '%s'" % tmp_backup_dir
exec_id = docker_obj.exec_create(self.group_id + '_' + instance_num,
cmd)
out = docker_obj.exec_start(exec_id)
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] != 0:
raise RuntimeError(
"Failed to create temp backup dir for container " +
instance_id)
for file_to_backup in files_to_backup:
cmd = "ln /var/lib/tarantool/%s %s/%s" % (
file_to_backup, tmp_backup_dir, file_to_backup)
exec_id = docker_obj.exec_create(
self.group_id + '_' + instance_num, cmd)
out = docker_obj.exec_start(exec_id)
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] != 0:
raise RuntimeError(
"Failed to hardlink backup file: " + out.decode('utf-8'))
strm, _ = docker_obj.get_archive(instance_id, tmp_backup_dir+'/.')
archive_id, size = storage.put_archive(strm)
cmd = "rm -rf /var/lib/tarantool/backup-*"
exec_id = docker_obj.exec_create(self.group_id + '_' + instance_num,
cmd)
out = docker_obj.exec_start(exec_id)
ret = docker_obj.exec_inspect(exec_id)
if ret['ExitCode'] != 0:
raise RuntimeError(
"Failed to remove temp backup dir for container " +
instance_id)
mem_used = services['instances'][instance_num]['mem_used']
storage.register_backup(backup_id, archive_id, group_id,
'memcached', size, mem_used)
Sense.update()
backup_task.set_status(task.STATUS_SUCCESS)
except Exception as ex:
logging.exception("Failed to backup '%s'", group_id)
backup_task.set_status(task.STATUS_CRITICAL, str(ex))