def __wrapper(task_self, self, **kwargs):
ret, meta, state = None, None, None
notify = 'Task Update Notify: {0}.'
params, main_task_id = kwargs, ''
if 'main_task' in kwargs:
main_task_id = kwargs['main_task'].task_id
params = {'main_task_id': main_task_id}
begin_time = datetime.now()
try:
# record task
columns = dict(task_id=task_self.request.id,
name=task_self.name,
state=C.TASK_STATE.PROGRESS.value,
kwargs=json.dumps(params),
begin_time=begin_time)
msg = notify.format(self.insert(**columns))
app.logger.info(logmsg(msg))
# update main task if exists
if main_task_id:
msg = notify.format(
self.update(task_id=main_task_id,
sub_task_id=task_self.request.id))
app.logger.info(logmsg(msg))
# excute task
result = func(task_self, self, **kwargs)
ret, meta, state = self.task_step(
task=task_self,
message=success_message.format(main_task_id),
data=result,
state=C.TASK_STATE.SUCCESS.value,
current=C.TASK_PERCENTAGE.ENDPOINT.value)
except Exception as e:
app.logger.error(logmsg(traceback.format_exc()))
ret, meta, state = self.task_step(
task=task_self,
message=failure_message.format(main_task_id),
error=str(e),
state=C.TASK_STATE.FAILURE.value,
current=C.TASK_PERCENTAGE.ENDPOINT.value)
finally:
# record task
end_time = datetime.now()
columns = dict(task_id=task_self.request.id,
info=json.dumps(meta),
end_time=end_time,
delta_time=(end_time -
begin_time).total_seconds(),
state=state)
msg = notify.format(self.update(**columns))
app.logger.info(logmsg(msg))
# update main task if exists
if main_task_id:
msg = notify.format(
self.update(
task_id=main_task_id,
ack_status=eval(
'C.CONFIGURATION_UPDATE_ACK_STATE.{0}.value'.
format(task_self.name))))
app.logger.info(logmsg(msg))
return state, meta
def delete_expired_files(self):
"""delete expired toml/tmpl files in remote confd client
"""
cfg_names = [x['name'] for x in self._files]
for host in self._hosts:
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
# 1. delete expired toml file
tomls = self.get_tomls(host=host)
for x in tomls:
config = x.split(self._file_pre)[1].split('toml')[0].strip('.')
if config not in cfg_names:
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='file',
args=dict(path=os.path.join(self._r_toml, x),
state='absent'))
msg = 'Toml File Deleted: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Toml File Deleted: %s' % results
app.logger.info(logmsg(msg))
# 2. delete expired tmpl file
tmpls = self.get_tmpls(host=host)
for x in tmpls:
config = x.split('.tmpl')[0]
if config not in cfg_names:
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='file',
args=dict(path=os.path.join(self._r_tmpl,
self._folder_pre, x),
state='absent'))
msg = 'Tmpl File Deleted: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Tmpl File Deleted: %s' % results
app.logger.info(logmsg(msg))
def get(self):
""" get whole list of tasks
"""
try:
try:
args = self.parser.parse_args()
except Exception as e:
if hasattr(e, 'data'):
raise SakuraInvalidAccessError(e.data['message'])
else:
raise SakuraInvalidAccessError()
pages, data, kwargs = False, [], {}
kwargs = {k: v for k, v in args.items() if k in self.params and v}
page = args['page']
if kwargs or not page:
data = self.obj.get(**kwargs)
else:
query = []
per_page = args['per_page']
if per_page:
query = self.obj.get(page=page, per_page=per_page, **kwargs)
else:
query = self.obj.get(page=page, **kwargs)
if query:
data, pages = query[0], query[1]
return {'totalpage': pages, 'data': data}, 200
except SakuraAPIError as e:
app.logger.error(logmsg(traceback.format_exc()))
return {'error': e.message, 'status': 1}, e.code
except Exception as e:
app.logger.error(logmsg(traceback.format_exc()))
return {'error': str(e), 'status': 1}, 500
def confd_cmd(self, action):
""" confd client startup cmd """
aapi = Ansible2API(hosts=self._hosts, **self._ansible_kwargs)
state, state_sum, results = ansible_safe_run(
aapi=aapi, module='shell', args=self._confd_startup_cmd[action])
msg = 'Confd %s: %s' % (action.upper(), state_sum)
app.logger.debug(logmsg(msg))
msg = 'Confd %s: %s' % (action.upper(), results)
app.logger.info(logmsg(msg))
def get_tmpl_content(self, cfg_name, host):
"""get template file content from remote confd client
"""
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='shell',
args='cat %s.tmpl' % os.path.join(self._folder_pre, cfg_name))
msg = 'Tmpl File Get: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Tmpl File Get: %s' % results
app.logger.info(logmsg(msg))
return '%s\r\n' % results[host]['stdout']
def get_tmpls(self, host):
"""get template files from remote confd client
"""
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='shell',
args='ls %s | awk 1' %
os.path.join(self._r_tmpl, self._folder_pre))
msg = 'Tmpl File Check: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Tmpl File Check: %s' % results
app.logger.info(logmsg(msg))
return results[host]['stdout_lines']
def get_tomls(self, host):
"""get toml files from remote confd client
"""
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='shell',
args='ls %s | grep "^%s\..*\.toml$" | awk 1' %
(self._r_toml, self._file_pre))
msg = 'Toml File Check: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Toml File Check: %s' % results
app.logger.info(logmsg(msg))
return results[host]['stdout_lines']
def post(self):
""" execute a task
"""
try:
args = self._parse_args(params=self.post_params)
args = self._before_task(args=args)
task = eval('self.task.%s' % self.task_name).apply_async(kwargs=args)
return {'id': task.task_id, 'status': 0}, 201
except SakuraAPIError as e:
app.logger.error(logmsg(traceback.format_exc()))
return {'error': e.message, 'status': 1}, e.code
except Exception as e:
app.logger.error(logmsg(traceback.format_exc()))
return {'error': str(e), 'status': 1}, 500
def create_toml(self):
"""create toml files
"""
result = {}
for x in self._files:
result[x['name']] = {}
usr = self.get_uids(**x['owner'])
for host in self._hosts:
toml_file = os.path.join(
self._l_toml, host,
'{0}.{1}.toml'.format(self._file_pre, x['name']))
with open(toml_file, 'w') as f:
content = [
'[template]\r\n', 'prefix = "/{0}"\r\n'.format(
os.path.join(self._folder_pre, x['name'])),
'keys = {0}\r\n'.format(
[str(k)
for k in x['items'].keys()] if x['items'] else []
), 'src = "{0}.tmpl"\r\n'.format(
os.path.join(self._folder_pre, x['name'])),
'dest = "{0}"\r\n'.format(
os.path.join(x['dir'], x['name'])),
'uid = {0}\r\n'.format(usr[host]['uid']),
'gid = {0}\r\n'.format(usr[host]['gid']),
'mode = "{0}"\r\n'.format(x['mode']),
'check_cmd = "{0}"\r\n'.format(
self._check_cmd.replace('"', '\'')),
'reload_cmd = "{0}"\r\n'.format(
self._reload_cmd.replace('"', '\''))
]
f.writelines(content)
msg = 'Toml File Created: %s.' % toml_file
app.logger.info(logmsg(msg))
result[x['name']][host] = toml_file
return result
def get(self, id):
""" check a specific task status
"""
try:
task = eval('self.task.%s' % self.task_name).AsyncResult(id)
result = {
'id': task.id,
'state': task.result[0] if task.ready() else task.state,
'info': task.result[1] if task.ready() else task.result
}
return {'result': result, 'status': 0}, 200
except SakuraAPIError as e:
app.logger.error(logmsg(traceback.format_exc()))
return {'error': e.message, 'status': 1}, e.code
except Exception as e:
app.logger.error(logmsg(traceback.format_exc()))
return {'error': str(e), 'status': 1}, 500
def get_toml_content(self, cfg_name, host):
"""get toml file content from remote confd client
"""
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='shell',
args='cat %s.%s.toml' % (self._file_pre, cfg_name))
msg = 'Toml File Get: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Toml File Get: %s' % results
app.logger.info(logmsg(msg))
results = results[host]['stdout_lines'][1:]
ret = dict()
for x in results:
key, value = x.split(' = ')
ret[key] = value.strip('\"')
return ret
def insert(self, **kwargs):
if kwargs:
obj = self.__class__(**kwargs)
db.session.add(obj)
try:
db.session.commit()
return obj._to_dict()
except Exception, e:
db.session.rollback()
msg = self.__DBContraintException % e
app.logger.error(logmsg(msg))
return {}
def delete_expired_keys(self):
"""delete expired configuration keys stored in etcd server
"""
for x in self._files:
key_pre = os.path.join('/', self._folder_pre, x['name'])
if key_pre in self.etcd:
father = self.etcd.read(key=key_pre)
if hasattr(father, '_children'):
for y in father._children:
if y['key'].split('/')[-1] not in x['items'].keys():
ret = self.etcd.delete(key=y['key'])
msg = 'Etcd Key Deleted: %s.' % ret
app.logger.info(logmsg(msg))
def update_keys(self, rollback=False):
""" update configuration keys stored in etcd server
ps: when called for rollback, would delete keys totally new
"""
for x in self._files:
items = (self.get_keys(cfg_name=x['name'], rollback=rollback)
if rollback else x['items'])
# delete keys which did not exist before update when doing rollback
diff = set(x['items'].keys()).difference(set(items.keys()))
for k in diff:
key = os.path.join('/', self._folder_pre, x['name'], k)
if key in self.etcd:
ret = self.etcd.delete(key=key)
msg = 'Etcd Key Deleted: %s.' % ret
app.logger.info(logmsg(msg))
# update keys
for k, v in items.items():
ret = self.etcd.write(key=os.path.join('/', self._folder_pre,
x['name'], k),
value=v)
msg = 'Etcd Key Updated: %s.' % ret
app.logger.info(logmsg(msg))
def backup_keys(self):
"""backup configuration keys using etcd server
"""
dir_pre = os.path.join('/', self._key_bak_pre, self._folder_pre)
if dir_pre in self.etcd:
self.etcd.delete(key=dir_pre, dir=True, recursive=True)
for x in self._files:
items = self.get_keys(cfg_name=x['name'])
for k, v in items.items():
ret = self.etcd.write(key=os.path.join(dir_pre, x['name'], k),
value=v)
msg = 'Etcd Key Backup: %s.' % ret
app.logger.info(logmsg(msg))
def create_tmpl(self):
"""create template files
"""
result = {}
for x in self._files:
tmpl_file = '{0}.tmpl'.format(os.path.join(self._l_tmpl,
x['name']))
with open(tmpl_file, 'w') as f:
f.write(x['template'].encode('utf-8'))
msg = 'Tmpl File Created: %s.' % tmpl_file
app.logger.info(logmsg(msg))
result[x['name']] = tmpl_file
return result
def task_step(
self, task, name=None, args=None,
state=C.TASK_STATE.PROGRESS.value,
current=C.TASK_PERCENTAGE.STARTPOINT.value,
data=None, message=None, error=None, flag=None):
if flag:
msg = self.update(task_id=task.request.id, step=flag)
app.logger.debug(logmsg(msg))
args = args if args else {}
meta = dict(
current=current,
total=C.TASK_PERCENTAGE.ENDPOINT.value,
message=message,
data=data)
if error:
meta['error'] = error
app.logger.info(logmsg(state))
app.logger.info(logmsg(meta))
task.update_state(task_id=task.request.id, state=state, meta=meta)
ret = name(**args) if name else None
app.logger.debug(logmsg(ret))
return ret, meta, state
def update(self, task_id, **kwargs):
obj = self.__class__.query.filter_by(task_id=task_id).first()
if obj:
for k, v in kwargs.items():
if k in self._columns():
setattr(obj, k, v)
try:
db.session.commit()
return obj._to_dict()
except Exception, e:
db.session.rollback()
msg = self.__DBContraintException % e
app.logger.error(logmsg(msg))
return {}
def get_uids(self, name, group):
"""fetch owner's uid and gid via name
"""
aapi = Ansible2API(hosts=self._hosts, **self._ansible_kwargs)
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='shell',
args="cat /etc/passwd | grep ^%s: | awk -F ':' '{print $3}';"
"cat /etc/passwd | grep ^%s: | awk -F ':' '{print $3}';" %
(name, group))
msg = 'Uid and Gid Get: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Uid and Gid Get: %s' % results
app.logger.info(logmsg(msg))
ret = {}
for k, v in results.items():
if len(v['stdout_lines']) == 2:
ret[k] = dict(uid=v['stdout_lines'][0],
gid=v['stdout_lines'][1])
else:
raise Exception('{0}: No such user or group ({1}, {2})'.format(
k, name, group))
return ret
def get_keys(self, cfg_name, rollback=False):
""" get configuration keys stored in the etcd server """
key_pre = os.path.join(
'/', (os.path.join(self._key_bak_pre, self._folder_pre)
if rollback else os.path.join(self._folder_pre)), cfg_name)
items = {}
if key_pre in self.etcd:
father = self.etcd.read(key=key_pre)
msg = 'Etcd Key Read: %s.' % father
app.logger.info(logmsg(msg))
if hasattr(father, '_children'):
for x in father._children:
items[x['key'].split('/')[-1]] = x['value']
return items
def delete_files(self):
"""
delete old toml/tmpl files in remote confd client
ps: make sure that all these files have been backup already
"""
for host in self._hosts:
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
# 1. delete toml
tomls = self.get_tomls(host=host)
for x in tomls:
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='file',
args=dict(path=os.path.join(self._r_toml, x),
state='absent'))
msg = 'Toml File Deleted: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Toml File Deleted: %s' % results
app.logger.info(logmsg(msg))
# 2. delete tmpl
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='file',
args=dict(path='%s/' %
os.path.join(self._r_tmpl, self._folder_pre),
state='absent'))
msg = 'Tmpl File Deleted: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Tmpl File Deleted: %s' % results
app.logger.info(logmsg(msg))
# 3. delete conf
for x in self._files:
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='file',
args=dict(path=os.path.join(x['dir'], x['name']),
state='absent'))
msg = 'Conf File Deleted: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Conf File Deleted: %s' % results
app.logger.info(logmsg(msg))
def backup_files(self):
"""backup old toml/tmpl/cfg files from remote confd client to server
"""
for host in self._hosts:
# local filesystem
toml_bak = os.path.join(self._l_toml_bak, host)
tmpl_bak = os.path.join(self._l_tmpl_bak, host)
conf_bak = os.path.join(self._l_conf_bak, host)
remove_folder(toml_bak)
remove_folder(tmpl_bak)
remove_folder(conf_bak)
get_folder(toml_bak)
get_folder(tmpl_bak)
get_folder(conf_bak)
# minio server
toml_pre = '%s/' % os.path.join('toml', self._folder_pre, host)
tmpl_pre = '%s/' % os.path.join('tmpl', self._folder_pre, host)
conf_pre = '%s/' % os.path.join('conf', self._folder_pre, host)
objs = self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=toml_pre,
recursive=False)
for x in objs:
self.minio.remove_object(
bucket_name=self._minio_bucket,
object_name=x.object_name.encode('utf-8'))
objs = self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=tmpl_pre,
recursive=False)
for x in objs:
self.minio.remove_object(
bucket_name=self._minio_bucket,
object_name=x.object_name.encode('utf-8'))
objs = self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=conf_pre,
recursive=False)
for x in objs:
self.minio.remove_object(
bucket_name=self._minio_bucket,
object_name=x.object_name.encode('utf-8'))
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
# 1. backup toml to minio server
tomls = self.get_tomls(host=host)
for x in tomls:
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='fetch',
args=dict(dest='%s/' % toml_bak,
src=os.path.join(self._r_toml, x),
flat='yes'))
msg = 'Toml File Backup: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Toml File Backup: %s' % results
app.logger.info(logmsg(msg))
self.minio.fput_object(bucket_name=self._minio_bucket,
object_name=os.path.join(toml_pre, x),
file_path=os.path.join(toml_bak, x))
# 2. backup tmpl to minio server
tmpls = self.get_tmpls(host=host)
for x in tmpls:
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='fetch',
args=dict(dest='%s/' % tmpl_bak,
src=os.path.join(self._r_tmpl, self._folder_pre,
x),
flat='yes'))
msg = 'Tmpl File Backup: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Tmpl File Backup: %s' % results
app.logger.info(logmsg(msg))
self.minio.fput_object(bucket_name=self._minio_bucket,
object_name=os.path.join(tmpl_pre, x),
file_path=os.path.join(tmpl_bak, x))
# 3. backup conf to minio server
# files should include (name, dir, mode, owner)
for x in self._files:
src = os.path.join(x['dir'], x['name'])
file_name = '%s%s%s' % ('@@'.join([
x['mode'], x['owner']['name'], x['owner']['group']
]), self._broken_word_2, src.replace('/', self._broken_word_1))
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='fetch',
args=dict(dest=os.path.join(conf_bak, file_name),
src=src,
flat='yes'))
msg = 'Conf File Backup: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Conf File Backup: %s' % results
app.logger.info(logmsg(msg))
file_path = os.path.join(conf_bak, file_name)
if os.path.isfile(file_path):
self.minio.fput_object(bucket_name=self._minio_bucket,
object_name=os.path.join(
conf_pre, file_name),
file_path=file_path)
# 4. check if toml/tmpl/conf have been backuped to minio server
objs = [
os.path.basename(x.object_name.encode('utf-8')) for x in
self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=toml_pre,
recursive=False)
]
for x in tomls:
if x not in objs:
raise Exception('Toml Backup Failed: %s.' % x)
objs = [
os.path.basename(x.object_name.encode('utf-8')) for x in
self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=tmpl_pre,
recursive=False)
]
for x in tmpls:
if x not in objs:
raise Exception('Tmpl Backup Failed: %s.' % x)
def push_files(self, rollback=False):
""" update toml/tmpl/(conf) files to remote/local confd client """
for host in self._hosts:
aapi = Ansible2API(hosts=[host], **self._ansible_kwargs)
toml_folder = '%s/' % (os.path.join(self._l_toml_bak, host)
if rollback else os.path.join(
self._l_toml, host))
tmpl_folder = '{}/'.format(
os.path.join(self._l_tmpl_bak, host) if rollback else self.
_l_tmpl)
if rollback:
conf_folder = '%s/' % os.path.join(self._l_conf_bak, host)
# clear folders
remove_folder(toml_folder)
remove_folder(tmpl_folder)
remove_folder(conf_folder)
get_folder(toml_folder)
get_folder(tmpl_folder)
get_folder(conf_folder)
# download latest tomls/tmpls from minio
toml_pre = '%s/' % os.path.join('toml', self._folder_pre, host)
objs = self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=toml_pre,
recursive=False)
for x in objs:
object_name = x.object_name.encode('utf-8')
self.minio.fget_object(bucket_name=self._minio_bucket,
object_name=object_name,
file_path=os.path.join(
toml_folder,
os.path.basename(object_name)))
tmpl_pre = '%s/' % os.path.join('tmpl', self._folder_pre, host)
objs = self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=tmpl_pre,
recursive=False)
for x in objs:
object_name = x.object_name.encode('utf-8')
self.minio.fget_object(bucket_name=self._minio_bucket,
object_name=object_name,
file_path=os.path.join(
tmpl_folder,
os.path.basename(object_name)))
conf_pre = '%s/' % os.path.join('conf', self._folder_pre, host)
objs = self.minio.list_objects(bucket_name=self._minio_bucket,
prefix=conf_pre,
recursive=False)
for x in objs:
object_name = x.object_name.encode('utf-8')
self.minio.fget_object(bucket_name=self._minio_bucket,
object_name=object_name,
file_path=os.path.join(
conf_folder,
os.path.basename(object_name)))
# push conf files to remote/local confd client
for x in os.listdir(conf_folder):
config = x.split(self._broken_word_2)
file_path = config[1].replace(self._broken_word_1, '/')
info = config[0].split('@@')
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='copy',
args=dict(mode=info[0],
src=os.path.join(conf_folder, x),
dest=file_path,
group=info[2],
owner=info[1]))
msg = 'Conf File Updated: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Conf File Updated: %s' % results
app.logger.info(logmsg(msg))
# 1. push toml files to remote/local confd client
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='copy',
args=dict(mode=self._confd_file_mode,
src=toml_folder,
dest=self._r_toml,
group=self._confd_owner[1],
owner=self._confd_owner[0]))
msg = 'Toml File Updated: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Toml File Updated: %s' % results
app.logger.info(logmsg(msg))
# 2. push tmpl files to remote/local confd client
r_tmpl_folder = os.path.join(self._r_tmpl, self._folder_pre)
state, state_sum, results = ansible_safe_run(
aapi=aapi,
module='copy',
args=dict(mode=self._confd_file_mode,
src=tmpl_folder,
dest=r_tmpl_folder,
group=self._confd_owner[1],
owner=self._confd_owner[0]))
msg = 'Tmpl File Updated: %s' % state_sum
app.logger.debug(logmsg(msg))
msg = 'Tmpl File Updated: %s' % results
app.logger.info(logmsg(msg))