def __wrapper(task_self, self, **kwargs): ret, meta, state = None, None, None notify = 'Task Update Notify: {0}.' params, main_task_id = kwargs, '' if 'main_task' in kwargs: main_task_id = kwargs['main_task'].task_id params = {'main_task_id': main_task_id} begin_time = datetime.now() try: # record task columns = dict(task_id=task_self.request.id, name=task_self.name, state=C.TASK_STATE.PROGRESS.value, kwargs=json.dumps(params), begin_time=begin_time) msg = notify.format(self.insert(**columns)) app.logger.info(logmsg(msg)) # update main task if exists if main_task_id: msg = notify.format( self.update(task_id=main_task_id, sub_task_id=task_self.request.id)) app.logger.info(logmsg(msg)) # excute task result = func(task_self, self, **kwargs) ret, meta, state = self.task_step( task=task_self, message=success_message.format(main_task_id), data=result, state=C.TASK_STATE.SUCCESS.value, current=C.TASK_PERCENTAGE.ENDPOINT.value) except Exception as e: app.logger.error(logmsg(traceback.format_exc())) ret, meta, state = self.task_step( task=task_self, message=failure_message.format(main_task_id), error=str(e), state=C.TASK_STATE.FAILURE.value, current=C.TASK_PERCENTAGE.ENDPOINT.value) finally: # record task end_time = datetime.now() columns = dict(task_id=task_self.request.id, info=json.dumps(meta), end_time=end_time, delta_time=(end_time - begin_time).total_seconds(), state=state) msg = notify.format(self.update(**columns)) app.logger.info(logmsg(msg)) # update main task if exists if main_task_id: msg = notify.format( self.update( task_id=main_task_id, ack_status=eval( 'C.CONFIGURATION_UPDATE_ACK_STATE.{0}.value'. format(task_self.name)))) app.logger.info(logmsg(msg)) return state, meta
def delete_expired_files(self): """delete expired toml/tmpl files in remote confd client """ cfg_names = [x['name'] for x in self._files] for host in self._hosts: aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) # 1. delete expired toml file tomls = self.get_tomls(host=host) for x in tomls: config = x.split(self._file_pre)[1].split('toml')[0].strip('.') if config not in cfg_names: state, state_sum, results = ansible_safe_run( aapi=aapi, module='file', args=dict(path=os.path.join(self._r_toml, x), state='absent')) msg = 'Toml File Deleted: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Toml File Deleted: %s' % results app.logger.info(logmsg(msg)) # 2. delete expired tmpl file tmpls = self.get_tmpls(host=host) for x in tmpls: config = x.split('.tmpl')[0] if config not in cfg_names: state, state_sum, results = ansible_safe_run( aapi=aapi, module='file', args=dict(path=os.path.join(self._r_tmpl, self._folder_pre, x), state='absent')) msg = 'Tmpl File Deleted: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Tmpl File Deleted: %s' % results app.logger.info(logmsg(msg))
def get(self): """ get whole list of tasks """ try: try: args = self.parser.parse_args() except Exception as e: if hasattr(e, 'data'): raise SakuraInvalidAccessError(e.data['message']) else: raise SakuraInvalidAccessError() pages, data, kwargs = False, [], {} kwargs = {k: v for k, v in args.items() if k in self.params and v} page = args['page'] if kwargs or not page: data = self.obj.get(**kwargs) else: query = [] per_page = args['per_page'] if per_page: query = self.obj.get(page=page, per_page=per_page, **kwargs) else: query = self.obj.get(page=page, **kwargs) if query: data, pages = query[0], query[1] return {'totalpage': pages, 'data': data}, 200 except SakuraAPIError as e: app.logger.error(logmsg(traceback.format_exc())) return {'error': e.message, 'status': 1}, e.code except Exception as e: app.logger.error(logmsg(traceback.format_exc())) return {'error': str(e), 'status': 1}, 500
def confd_cmd(self, action): """ confd client startup cmd """ aapi = Ansible2API(hosts=self._hosts, **self._ansible_kwargs) state, state_sum, results = ansible_safe_run( aapi=aapi, module='shell', args=self._confd_startup_cmd[action]) msg = 'Confd %s: %s' % (action.upper(), state_sum) app.logger.debug(logmsg(msg)) msg = 'Confd %s: %s' % (action.upper(), results) app.logger.info(logmsg(msg))
def get_tmpl_content(self, cfg_name, host): """get template file content from remote confd client """ aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) state, state_sum, results = ansible_safe_run( aapi=aapi, module='shell', args='cat %s.tmpl' % os.path.join(self._folder_pre, cfg_name)) msg = 'Tmpl File Get: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Tmpl File Get: %s' % results app.logger.info(logmsg(msg)) return '%s\r\n' % results[host]['stdout']
def get_tmpls(self, host): """get template files from remote confd client """ aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) state, state_sum, results = ansible_safe_run( aapi=aapi, module='shell', args='ls %s | awk 1' % os.path.join(self._r_tmpl, self._folder_pre)) msg = 'Tmpl File Check: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Tmpl File Check: %s' % results app.logger.info(logmsg(msg)) return results[host]['stdout_lines']
def get_tomls(self, host): """get toml files from remote confd client """ aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) state, state_sum, results = ansible_safe_run( aapi=aapi, module='shell', args='ls %s | grep "^%s\..*\.toml$" | awk 1' % (self._r_toml, self._file_pre)) msg = 'Toml File Check: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Toml File Check: %s' % results app.logger.info(logmsg(msg)) return results[host]['stdout_lines']
def post(self): """ execute a task """ try: args = self._parse_args(params=self.post_params) args = self._before_task(args=args) task = eval('self.task.%s' % self.task_name).apply_async(kwargs=args) return {'id': task.task_id, 'status': 0}, 201 except SakuraAPIError as e: app.logger.error(logmsg(traceback.format_exc())) return {'error': e.message, 'status': 1}, e.code except Exception as e: app.logger.error(logmsg(traceback.format_exc())) return {'error': str(e), 'status': 1}, 500
def create_toml(self): """create toml files """ result = {} for x in self._files: result[x['name']] = {} usr = self.get_uids(**x['owner']) for host in self._hosts: toml_file = os.path.join( self._l_toml, host, '{0}.{1}.toml'.format(self._file_pre, x['name'])) with open(toml_file, 'w') as f: content = [ '[template]\r\n', 'prefix = "/{0}"\r\n'.format( os.path.join(self._folder_pre, x['name'])), 'keys = {0}\r\n'.format( [str(k) for k in x['items'].keys()] if x['items'] else [] ), 'src = "{0}.tmpl"\r\n'.format( os.path.join(self._folder_pre, x['name'])), 'dest = "{0}"\r\n'.format( os.path.join(x['dir'], x['name'])), 'uid = {0}\r\n'.format(usr[host]['uid']), 'gid = {0}\r\n'.format(usr[host]['gid']), 'mode = "{0}"\r\n'.format(x['mode']), 'check_cmd = "{0}"\r\n'.format( self._check_cmd.replace('"', '\'')), 'reload_cmd = "{0}"\r\n'.format( self._reload_cmd.replace('"', '\'')) ] f.writelines(content) msg = 'Toml File Created: %s.' % toml_file app.logger.info(logmsg(msg)) result[x['name']][host] = toml_file return result
def get(self, id): """ check a specific task status """ try: task = eval('self.task.%s' % self.task_name).AsyncResult(id) result = { 'id': task.id, 'state': task.result[0] if task.ready() else task.state, 'info': task.result[1] if task.ready() else task.result } return {'result': result, 'status': 0}, 200 except SakuraAPIError as e: app.logger.error(logmsg(traceback.format_exc())) return {'error': e.message, 'status': 1}, e.code except Exception as e: app.logger.error(logmsg(traceback.format_exc())) return {'error': str(e), 'status': 1}, 500
def get_toml_content(self, cfg_name, host): """get toml file content from remote confd client """ aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) state, state_sum, results = ansible_safe_run( aapi=aapi, module='shell', args='cat %s.%s.toml' % (self._file_pre, cfg_name)) msg = 'Toml File Get: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Toml File Get: %s' % results app.logger.info(logmsg(msg)) results = results[host]['stdout_lines'][1:] ret = dict() for x in results: key, value = x.split(' = ') ret[key] = value.strip('\"') return ret
def insert(self, **kwargs): if kwargs: obj = self.__class__(**kwargs) db.session.add(obj) try: db.session.commit() return obj._to_dict() except Exception, e: db.session.rollback() msg = self.__DBContraintException % e app.logger.error(logmsg(msg)) return {}
def delete_expired_keys(self): """delete expired configuration keys stored in etcd server """ for x in self._files: key_pre = os.path.join('/', self._folder_pre, x['name']) if key_pre in self.etcd: father = self.etcd.read(key=key_pre) if hasattr(father, '_children'): for y in father._children: if y['key'].split('/')[-1] not in x['items'].keys(): ret = self.etcd.delete(key=y['key']) msg = 'Etcd Key Deleted: %s.' % ret app.logger.info(logmsg(msg))
def update_keys(self, rollback=False): """ update configuration keys stored in etcd server ps: when called for rollback, would delete keys totally new """ for x in self._files: items = (self.get_keys(cfg_name=x['name'], rollback=rollback) if rollback else x['items']) # delete keys which did not exist before update when doing rollback diff = set(x['items'].keys()).difference(set(items.keys())) for k in diff: key = os.path.join('/', self._folder_pre, x['name'], k) if key in self.etcd: ret = self.etcd.delete(key=key) msg = 'Etcd Key Deleted: %s.' % ret app.logger.info(logmsg(msg)) # update keys for k, v in items.items(): ret = self.etcd.write(key=os.path.join('/', self._folder_pre, x['name'], k), value=v) msg = 'Etcd Key Updated: %s.' % ret app.logger.info(logmsg(msg))
def backup_keys(self): """backup configuration keys using etcd server """ dir_pre = os.path.join('/', self._key_bak_pre, self._folder_pre) if dir_pre in self.etcd: self.etcd.delete(key=dir_pre, dir=True, recursive=True) for x in self._files: items = self.get_keys(cfg_name=x['name']) for k, v in items.items(): ret = self.etcd.write(key=os.path.join(dir_pre, x['name'], k), value=v) msg = 'Etcd Key Backup: %s.' % ret app.logger.info(logmsg(msg))
def create_tmpl(self): """create template files """ result = {} for x in self._files: tmpl_file = '{0}.tmpl'.format(os.path.join(self._l_tmpl, x['name'])) with open(tmpl_file, 'w') as f: f.write(x['template'].encode('utf-8')) msg = 'Tmpl File Created: %s.' % tmpl_file app.logger.info(logmsg(msg)) result[x['name']] = tmpl_file return result
def task_step( self, task, name=None, args=None, state=C.TASK_STATE.PROGRESS.value, current=C.TASK_PERCENTAGE.STARTPOINT.value, data=None, message=None, error=None, flag=None): if flag: msg = self.update(task_id=task.request.id, step=flag) app.logger.debug(logmsg(msg)) args = args if args else {} meta = dict( current=current, total=C.TASK_PERCENTAGE.ENDPOINT.value, message=message, data=data) if error: meta['error'] = error app.logger.info(logmsg(state)) app.logger.info(logmsg(meta)) task.update_state(task_id=task.request.id, state=state, meta=meta) ret = name(**args) if name else None app.logger.debug(logmsg(ret)) return ret, meta, state
def update(self, task_id, **kwargs): obj = self.__class__.query.filter_by(task_id=task_id).first() if obj: for k, v in kwargs.items(): if k in self._columns(): setattr(obj, k, v) try: db.session.commit() return obj._to_dict() except Exception, e: db.session.rollback() msg = self.__DBContraintException % e app.logger.error(logmsg(msg)) return {}
def get_uids(self, name, group): """fetch owner's uid and gid via name """ aapi = Ansible2API(hosts=self._hosts, **self._ansible_kwargs) state, state_sum, results = ansible_safe_run( aapi=aapi, module='shell', args="cat /etc/passwd | grep ^%s: | awk -F ':' '{print $3}';" "cat /etc/passwd | grep ^%s: | awk -F ':' '{print $3}';" % (name, group)) msg = 'Uid and Gid Get: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Uid and Gid Get: %s' % results app.logger.info(logmsg(msg)) ret = {} for k, v in results.items(): if len(v['stdout_lines']) == 2: ret[k] = dict(uid=v['stdout_lines'][0], gid=v['stdout_lines'][1]) else: raise Exception('{0}: No such user or group ({1}, {2})'.format( k, name, group)) return ret
def get_keys(self, cfg_name, rollback=False): """ get configuration keys stored in the etcd server """ key_pre = os.path.join( '/', (os.path.join(self._key_bak_pre, self._folder_pre) if rollback else os.path.join(self._folder_pre)), cfg_name) items = {} if key_pre in self.etcd: father = self.etcd.read(key=key_pre) msg = 'Etcd Key Read: %s.' % father app.logger.info(logmsg(msg)) if hasattr(father, '_children'): for x in father._children: items[x['key'].split('/')[-1]] = x['value'] return items
def delete_files(self): """ delete old toml/tmpl files in remote confd client ps: make sure that all these files have been backup already """ for host in self._hosts: aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) # 1. delete toml tomls = self.get_tomls(host=host) for x in tomls: state, state_sum, results = ansible_safe_run( aapi=aapi, module='file', args=dict(path=os.path.join(self._r_toml, x), state='absent')) msg = 'Toml File Deleted: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Toml File Deleted: %s' % results app.logger.info(logmsg(msg)) # 2. delete tmpl state, state_sum, results = ansible_safe_run( aapi=aapi, module='file', args=dict(path='%s/' % os.path.join(self._r_tmpl, self._folder_pre), state='absent')) msg = 'Tmpl File Deleted: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Tmpl File Deleted: %s' % results app.logger.info(logmsg(msg)) # 3. delete conf for x in self._files: state, state_sum, results = ansible_safe_run( aapi=aapi, module='file', args=dict(path=os.path.join(x['dir'], x['name']), state='absent')) msg = 'Conf File Deleted: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Conf File Deleted: %s' % results app.logger.info(logmsg(msg))
def backup_files(self): """backup old toml/tmpl/cfg files from remote confd client to server """ for host in self._hosts: # local filesystem toml_bak = os.path.join(self._l_toml_bak, host) tmpl_bak = os.path.join(self._l_tmpl_bak, host) conf_bak = os.path.join(self._l_conf_bak, host) remove_folder(toml_bak) remove_folder(tmpl_bak) remove_folder(conf_bak) get_folder(toml_bak) get_folder(tmpl_bak) get_folder(conf_bak) # minio server toml_pre = '%s/' % os.path.join('toml', self._folder_pre, host) tmpl_pre = '%s/' % os.path.join('tmpl', self._folder_pre, host) conf_pre = '%s/' % os.path.join('conf', self._folder_pre, host) objs = self.minio.list_objects(bucket_name=self._minio_bucket, prefix=toml_pre, recursive=False) for x in objs: self.minio.remove_object( bucket_name=self._minio_bucket, object_name=x.object_name.encode('utf-8')) objs = self.minio.list_objects(bucket_name=self._minio_bucket, prefix=tmpl_pre, recursive=False) for x in objs: self.minio.remove_object( bucket_name=self._minio_bucket, object_name=x.object_name.encode('utf-8')) objs = self.minio.list_objects(bucket_name=self._minio_bucket, prefix=conf_pre, recursive=False) for x in objs: self.minio.remove_object( bucket_name=self._minio_bucket, object_name=x.object_name.encode('utf-8')) aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) # 1. backup toml to minio server tomls = self.get_tomls(host=host) for x in tomls: state, state_sum, results = ansible_safe_run( aapi=aapi, module='fetch', args=dict(dest='%s/' % toml_bak, src=os.path.join(self._r_toml, x), flat='yes')) msg = 'Toml File Backup: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Toml File Backup: %s' % results app.logger.info(logmsg(msg)) self.minio.fput_object(bucket_name=self._minio_bucket, object_name=os.path.join(toml_pre, x), file_path=os.path.join(toml_bak, x)) # 2. backup tmpl to minio server tmpls = self.get_tmpls(host=host) for x in tmpls: state, state_sum, results = ansible_safe_run( aapi=aapi, module='fetch', args=dict(dest='%s/' % tmpl_bak, src=os.path.join(self._r_tmpl, self._folder_pre, x), flat='yes')) msg = 'Tmpl File Backup: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Tmpl File Backup: %s' % results app.logger.info(logmsg(msg)) self.minio.fput_object(bucket_name=self._minio_bucket, object_name=os.path.join(tmpl_pre, x), file_path=os.path.join(tmpl_bak, x)) # 3. backup conf to minio server # files should include (name, dir, mode, owner) for x in self._files: src = os.path.join(x['dir'], x['name']) file_name = '%s%s%s' % ('@@'.join([ x['mode'], x['owner']['name'], x['owner']['group'] ]), self._broken_word_2, src.replace('/', self._broken_word_1)) state, state_sum, results = ansible_safe_run( aapi=aapi, module='fetch', args=dict(dest=os.path.join(conf_bak, file_name), src=src, flat='yes')) msg = 'Conf File Backup: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Conf File Backup: %s' % results app.logger.info(logmsg(msg)) file_path = os.path.join(conf_bak, file_name) if os.path.isfile(file_path): self.minio.fput_object(bucket_name=self._minio_bucket, object_name=os.path.join( conf_pre, file_name), file_path=file_path) # 4. check if toml/tmpl/conf have been backuped to minio server objs = [ os.path.basename(x.object_name.encode('utf-8')) for x in self.minio.list_objects(bucket_name=self._minio_bucket, prefix=toml_pre, recursive=False) ] for x in tomls: if x not in objs: raise Exception('Toml Backup Failed: %s.' % x) objs = [ os.path.basename(x.object_name.encode('utf-8')) for x in self.minio.list_objects(bucket_name=self._minio_bucket, prefix=tmpl_pre, recursive=False) ] for x in tmpls: if x not in objs: raise Exception('Tmpl Backup Failed: %s.' % x)
def push_files(self, rollback=False): """ update toml/tmpl/(conf) files to remote/local confd client """ for host in self._hosts: aapi = Ansible2API(hosts=[host], **self._ansible_kwargs) toml_folder = '%s/' % (os.path.join(self._l_toml_bak, host) if rollback else os.path.join( self._l_toml, host)) tmpl_folder = '{}/'.format( os.path.join(self._l_tmpl_bak, host) if rollback else self. _l_tmpl) if rollback: conf_folder = '%s/' % os.path.join(self._l_conf_bak, host) # clear folders remove_folder(toml_folder) remove_folder(tmpl_folder) remove_folder(conf_folder) get_folder(toml_folder) get_folder(tmpl_folder) get_folder(conf_folder) # download latest tomls/tmpls from minio toml_pre = '%s/' % os.path.join('toml', self._folder_pre, host) objs = self.minio.list_objects(bucket_name=self._minio_bucket, prefix=toml_pre, recursive=False) for x in objs: object_name = x.object_name.encode('utf-8') self.minio.fget_object(bucket_name=self._minio_bucket, object_name=object_name, file_path=os.path.join( toml_folder, os.path.basename(object_name))) tmpl_pre = '%s/' % os.path.join('tmpl', self._folder_pre, host) objs = self.minio.list_objects(bucket_name=self._minio_bucket, prefix=tmpl_pre, recursive=False) for x in objs: object_name = x.object_name.encode('utf-8') self.minio.fget_object(bucket_name=self._minio_bucket, object_name=object_name, file_path=os.path.join( tmpl_folder, os.path.basename(object_name))) conf_pre = '%s/' % os.path.join('conf', self._folder_pre, host) objs = self.minio.list_objects(bucket_name=self._minio_bucket, prefix=conf_pre, recursive=False) for x in objs: object_name = x.object_name.encode('utf-8') self.minio.fget_object(bucket_name=self._minio_bucket, object_name=object_name, file_path=os.path.join( conf_folder, os.path.basename(object_name))) # push conf files to remote/local confd client for x in os.listdir(conf_folder): config = x.split(self._broken_word_2) file_path = config[1].replace(self._broken_word_1, '/') info = config[0].split('@@') state, state_sum, results = ansible_safe_run( aapi=aapi, module='copy', args=dict(mode=info[0], src=os.path.join(conf_folder, x), dest=file_path, group=info[2], owner=info[1])) msg = 'Conf File Updated: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Conf File Updated: %s' % results app.logger.info(logmsg(msg)) # 1. push toml files to remote/local confd client state, state_sum, results = ansible_safe_run( aapi=aapi, module='copy', args=dict(mode=self._confd_file_mode, src=toml_folder, dest=self._r_toml, group=self._confd_owner[1], owner=self._confd_owner[0])) msg = 'Toml File Updated: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Toml File Updated: %s' % results app.logger.info(logmsg(msg)) # 2. push tmpl files to remote/local confd client r_tmpl_folder = os.path.join(self._r_tmpl, self._folder_pre) state, state_sum, results = ansible_safe_run( aapi=aapi, module='copy', args=dict(mode=self._confd_file_mode, src=tmpl_folder, dest=r_tmpl_folder, group=self._confd_owner[1], owner=self._confd_owner[0])) msg = 'Tmpl File Updated: %s' % state_sum app.logger.debug(logmsg(msg)) msg = 'Tmpl File Updated: %s' % results app.logger.info(logmsg(msg))