def delete_task(_id, state):
record = db.collection('tasks').find_one({'t_id': _id})
if not record:
return jsonify({
'message': 'task not found',
'code': 194041
}), 404
task_id = record.get('t_id')
queue = record.get('queue')
try:
task = Task.from_id(tiger, queue, state, task_id)
task._move(from_state=state)
db.collection('tasks').update_one({'_id': record['_id']}, {'$set': {
'updated_at': datetime.now(),
'state': 'delete',
}})
extra = {
'queue': queue,
'task_id': task_id,
'from_state': state,
'to_state': None
}
logger.info('cancel task', extra=extra)
except TaskNotFound:
return jsonify({
'message': 'invalid task',
'code': 104044
}), 404
return jsonify({
'message': 'ok',
'code': 0,
})
def retry(_id, state):
record = db.collection('tasks').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'task not found',
'code': 194041
}), 404
task_id = record.get('t_id')
queue = record.get('queue')
try:
task = Task.from_id(tiger, queue, state, task_id)
task.retry()
db.collection('tasks').update_one({'_id': record['_id']}, {'$set': {
'updated_at': datetime.now(),
'state': state,
}})
extra = {
'queue': queue,
'task_id': task_id,
'from_state': state,
'to_state': QUEUED
}
logger.info('retry task', extra=extra)
except TaskNotFound:
return jsonify({
'message': 'invalid task',
'code': 104044
}), 404
return jsonify({
'message': 'ok',
'code': 0,
})
def get_job_tasks(_id):
query = request.args
job = db.collection('tasks').find_one({'_id': ObjectId(_id)})
if not job:
return jsonify({
'message': 'job not found',
'code': 194040
}), 404
status = query.get('status')
page = int(query.get('page', 1))
size = int(query.get('size', 25))
offset = (page - 1) * size
where = {
'job_id': _id,
}
if status:
where['status'] = status
cursor = db.collection('tasks').find(where, limit=size, skip=offset)
total = cursor.count()
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'list': list(cursor),
'total': total,
'page': page,
'pageSize': size
}
})
def send(self, text, user=None):
if not self.enable:
return False
user = user or '@all'
params = {
'agent_id': self.agent_id,
'text': text,
'user': user,
}
data = params.copy()
data['task_id'] = self.task_id
data['created_at'] = time.time()
try:
result = self.client.message.send_text(self.agent_id, user, text)
if not result:
raise Exception('send nexmo message with uncaught exception')
else:
data['code'] = 0
data['error'] = False
data['result'] = result
db.collection('alerts').insert_one(data)
return result
except Exception as err:
data['code'] = err.args[0] if type(err.args[0]) == int else -1
data['error'] = True
data['result'] = str(err.args[1])
data['trace'] = traceback.format_exc(limit=20)
db.collection('alerts').insert_one(data)
return False
def get_task_info(_id):
record = db.collection('tasks').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040
}), 404
log = db.collection('logs').find_one({'task_id': str(record.get('_id'))})
# log = db.collection('logs').find_one({'task_id': '5d6d4e0ae3f7e086eaa30321'})
record['log'] = log
job = db.collection('jobs').find_one({'_id': ObjectId(record.get('job_id'))})
record['job'] = job
queue = record.get('queue')
state = record.get('state')
task_id = record.get('t_id')
try:
task = Task.from_id(tiger, queue, state, task_id)
record['queue_info'] = task.data.copy()
except TaskNotFound:
record['queue_info'] = None
return jsonify({
'message': 'ok',
'code': 0,
'data': record,
})
def store_modules(self):
self.update_context_args({'json_dump': True})
result = self.run()
for key, value in result.items():
if not value.get('name'):
for k, v in value.items():
v['main'] = key
v['parent'] = k
v['created_at'] = time.time()
where = {
'name': v['name']
}
update = {
'$set': v
}
db.collection('ansible_modules').update_one(where, update=update, upsert=True)
else:
value['main'] = key
value['parent'] = key
value['created_at'] = time.time()
where = {
'name': value['name']
}
update = {
'$set': value
}
db.collection('ansible_modules').update_one(where, update=update, upsert=True)
return result
def get_roles_by_book(_id):
record = Book.find_one(({'_id': ObjectId(_id)}))
if not record:
return jsonify({
'message': 'book not found',
'code': '104001',
}), 400
book_id = str(record['_id'])
check_playbook(book_id)
condition = {'book_id': book_id, 'role': 'roles', 'is_dir': True}
parent = db.collection('playbook').find_one(condition)
if not parent:
return jsonify({
'message': 'ok',
'code': 0,
'data': [],
})
where = {'book_id': book_id, 'is_dir': True, 'parent': parent.get('path')}
cursor = db.collection('playbook').find(where)
records = list(cursor)
return jsonify({
'message': 'ok',
'code': 0,
'data': records,
})
def v2_runner_on_unreachable(self, result):
# super(CallbackModule, self).v2_runner_on_unreachable(result)
self._display.display("%s | UNREACHABLE! => %s"
% (result._host.get_name(), self._dump_results(result._result, indent=4)), color=None)
hostname = result._host.get_name()
vars = result._host.get_vars()
ansible_ssh_host = vars.get('ansible_ssh_host')
if ansible_ssh_host:
update = {
'$set': {
'state': 'unreachable',
'updated_at': datetime.datetime.now()
}
}
db.collection('machines').update_one({'ansible_ssh_host': ansible_ssh_host}, update=update)
self.host_unreachable[hostname] = result
# self.formatter.v2_runner_on_unreachable(result)
dumper = self.get_result(result)
dumper.update({'state': 'unreachable'})
self.dumper[hostname] = dumper
notification = 'ansible run on unreachable, host:{}'.format(hostname)
if self.job_id:
notification = notification + ',job name:'.format(self.job.get('name'))
self.notify(notification)
def host_tree():
user_id = login_user.get('user_id')
is_admin = login_user.get('is_admin')
if is_admin:
groups = db.collection('groups').find({})
group_ids = map(lambda i: i['_id'], groups)
group_ids = list(group_ids)
condition = [{
'$group': {
'_id': '$type',
'count': {
'$sum': 1
}
},
'$match': {
'user_id': user_id,
}
}]
result = db.collection('user_hosts').aggregate(condition)
return jsonify({
'message': 'ok',
'code': 0,
'data': [],
})
def add_user_to_team():
payload = request.get_json()
if not payload:
return jsonify({'message': 'invalid params', 'code': 104000}), 400
user_id = payload.get('user_id')
team_id = payload.get('team_id')
is_owner = payload.get('is_owner')
team = Team().find_by_id(user_id)
if not team:
return jsonify({'message': 'invalid team', 'code': 104031}), 400
where = {
'team_id': team_id,
'user_id': user_id,
'is_owner': is_owner,
}
data = where.copy()
data['created_at'] = time.time()
data['add_by'] = login_user.get('username')
db.collection('team_members').update_one(where, {'$set': data},
upsert=True)
role = db.collection('roles').find_one({
'name': team.get('name'),
'type': 'team',
})
return jsonify({
'message': 'ok',
'code': 0,
})
def add_admin(username, password):
has_admin = db.collection('users').find_one({'username': username})
if has_admin:
return True
admin = {
'email': '*****@*****.**',
'username': username,
'is_admin': True,
'nickname': 'administrator',
'phone': '+8613456789012',
'wechat': 'xx',
'address': '在那遥远的地方',
'email_status': 1,
'alerts': {
'task': [
'smtp',
'slack',
'web'
],
'api': [
'wechat',
'slack'
]
}
}
pwd = generate_password_hash(password)
admin['password'] = pwd
admin['created_at'] = time.time()
db.collection('users').insert_one(admin)
def update_group(_id):
where = {'_id': ObjectId(_id)}
record = db.collection('groups').find_one(where)
if not record:
return jsonify({'message': 'record not found', 'code': '124040'}), 404
payload = request.get_json()
if not payload:
return jsonify({
'message': 'miss required params',
'code': 124001
}), 400
data = dict()
for key, value in payload.items():
if record.get('key'):
data[key] = value
db.collection('regions').update_one(where, {'$set', data})
logger.info('update group', extra={'record': record, 'changed': data})
return jsonify({
'message': 'ok',
'code': 0,
})
def emit(self, record):
"""Inserting new logging record to mongo database."""
try:
data = self.format(record)
db.collection('logs').insert_one(data)
except Exception:
self.handleError(record)
def update_region(_id):
where = {'_id': ObjectId(_id)}
record = db.collection('regions').find_one(where)
if not record:
return jsonify({'message': 'record not found', 'code': '124040'}), 404
payload = request.get_json()
if not payload:
return jsonify({
'message': 'miss required params',
'code': 124001
}), 400
data = dict()
allow_fields = [
'platform', 'ip_range', 'bandwidth', 'contact', 'name', 'description'
]
for key, value in payload.items():
if key in allow_fields:
data[key] = value
db.collection('regions').update_one(where, {'$set': data})
data['_id'] = _id
logger.info('update region', extra={'record': record, 'changed': data})
return jsonify({
'message': 'ok',
'code': 0,
})
def bind_role(user_id):
payload = request.get_json()
if not payload:
return jsonify({'message': 'invalid params', 'code': 104030}), 400
role_ids = payload.get('role_ids')
if not role_ids or type(role_ids) != list:
return jsonify({'message': 'invalid params', 'code': 104031}), 400
user = User()
user_info = user.find_by_id(user_id)
if not user_info:
return jsonify({'message': 'record not found', 'code': 104040}), 404
roles = Role().find_by_ids(role_ids)
if not roles:
return jsonify({'message': 'invalid param', 'code': 104031}), 400
for role in roles:
data = {
'user_id': user_id,
'role_id': str(role['_id']),
}
where = data.copy()
data['created_at'] = time.time()
data['add_by'] = login_user.get('username')
db.collection('user_roles').update_one(where, {'$set': data},
upsert=True)
return jsonify({
'message': 'ok',
'code': 0,
})
def get_permissions(user_id):
user = db.collection('users').find_one({'_id': ObjectId(user_id)})
if not user:
return jsonify({'message': 'user not found', 'code': 104040}), 404
team = Team()
relate_team = db.collection('team_users').find({'user_id': user_id})
team_ids = list(map(lambda i: i['_id'], relate_team))
role_ids = []
permissions = []
if team_ids:
team_records = team.find_by_ids(team_ids)
for record in team_records:
team_role = db.collection('roles').find_one({
'name':
record.get('name'),
'type':
'team',
})
if not team_role:
continue
role_ids.append(team_role.get('_id'))
roles = db.collection('user_roles').find({'user_id': user_id})
roles = list(roles)
if roles:
ids = map(lambda i: i['_id'], roles)
role_ids.extend(list(ids))
if role_ids:
where = {'role_id': {'$in': role_ids}}
permissions = db.collection('role_menus').find(where)
permissions = list(permissions)
return jsonify({'message': 'ok', 'code': 0, 'data': permissions})
def send_verify_mail():
user_id = login_user.get('user_id')
record = User().find_by_id(user_id)
if not record:
return jsonify({
'message': 'invalid user',
'code': 104033
}), 403
email = record.get('email')
token = md5(str(current_request_id))
url = config.dommain + '/users/email/verify?token=' + token
message = '[Eclogue]Please click url to verify your email:<a href="{}">{}</a>'.format(url, url)
smtp = SMTP()
smtp.send(message, email, subtype='html')
data = {
'user_id': user_id,
'token': token,
'created_at': time.time(),
'email': email,
'content': message,
}
db.collection('mail_verify').insert_one(data)
return jsonify({
'message': 'ok',
'code': 0
})
def groups():
query = request.args or {}
keyword = query.get('keyword')
start = query.get('start')
end = query.get('end')
page = int(query.get('page', 1))
limit = int(query.get('pageSize', 50))
region = query.get('region')
where = {'status': {'$ne': -1}}
if region:
where['region'] = query.get('region')
if keyword:
where['name'] = {'$regex': keyword}
date = []
if start:
date.append({
'created_at': {
'$gte': int(time.mktime(time.strptime(start, '%Y-%m-%d')))
}
})
if end:
date.append({
'created_at': {
'$lte': int(time.mktime(time.strptime(end, '%Y-%m-%d')))
}
})
if date:
where['$and'] = date
offset = (page - 1) * limit
is_all = query.get('all')
if is_all and login_user.get('is_admin'):
records = db.collection('groups').find({})
else:
records = db.collection('groups').find(where, limit=limit, skip=offset)
total = records.count()
records = list(records)
for group in records:
region = db.collection('regions').find_one(
{'_id': ObjectId(group['region'])})
group['region_name'] = region.get('name')
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'list': list(records),
'page': page,
'pagesize': limit,
'total': total
}
})
def dowload_artifact(self, project_id, job_id, store):
# project_id = '13539397'
# job_id = '261939258'
is_cache = self.config.get('cache')
save_dir = os.path.dirname(store)
if is_cache:
cache_files = db.collection('artifacts').find({
'project_id': project_id,
'job_id': job_id,
'app_type': 'gitlab'
})
cache_files = list(cache_files)
if cache_files:
for record in cache_files:
file_id = record.get('file_id')
if not file_id:
continue
filename = os.path.join(save_dir, record['filename'])
with open(filename, 'wb') as stream:
db.fs_bucket().download_to_stream(file_id, stream)
extract(filename, save_dir)
os.unlink(filename)
return True
project = self.projects.get(project_id)
# pipeline = project.jobs.get(job_id)
jobs = project.jobs
job = jobs.get(job_id)
if job.status != 'success':
raise Exception('gitlab job status must be success, %s got'.format(job.status))
with open(store, 'wb') as f:
job.artifacts(streamed=True, action=f.write)
if is_cache:
with open(store, mode='rb') as fd:
name = os.path.basename(store)
file_id = db.save_file(filename=name, fileobj=fd)
store_info = {
'app_type': 'jenkins',
'file_id': str(file_id),
'project_id': project_id,
'job_id': job_id,
'filename': name,
'created_at': time.time()
}
db.collection('artifacts').insert_one(store_info)
with zipfile.ZipFile(store, 'r') as zip_ref:
zip_ref.extractall(os.path.dirname(store))
os.unlink(store)
return True
def check_task(self, book_name, options):
book = db.collection('books').find_one({'name': book_name})
if not book:
return False
inventory = db.collection('playbook').find_one({
'book_name':
book_name,
'name':
options.inventory
})
if not inventory:
return False
role = db.collection('playbook').find_one({
'book_name': book_name,
'name': options.role,
'role': 'roles'
})
if not role:
return False
book = self.load_book_from_db(book_name)
bucket = []
for key, item in book.items():
if item['role'] is 'entry' and not item['name'] is options.entry:
continue
if item['role'] is 'inventory' and not item[
'name'] is options.inventory:
continue
if item['role'] is 'tasks' and item['is_edit']:
content = []
if options.tags or options.skip_tags:
tasks = yaml.load_safe(item['content'])
for key, task in tasks.items():
task_tags = task.get('tags')
if not task:
continue
intersection = list(
set(options.tags).intersection(set(task_tags)))
if not intersection:
continue
intersection = list(
set(options.skip_tags).intersection(
set(task_tags)))
if intersection:
continue
content.append(task)
item.content = yaml.safe_dump(content)
item.pop('_id')
bucket.append(item)
if item['role'] is 'vars' and options.extra_vars:
variables = yaml.safe_load(item.content)
variables = variables.update(options.extra_vars)
item.content = yaml.safe_dump(variables)
return bucket
def web(self, user_id, msg_type, content):
record = {
'title': content[:20],
'content': content,
'user_id': user_id,
'action': None,
'params': None,
'read': 0,
'type': msg_type,
'created_at': time.time()
}
db.collection('notifications').insert_one(record)
def parse_cmdb_inventory(inventory):
if type(inventory) != list:
inventory = [inventory]
hosts = {}
data = {}
for inventory_str in inventory:
inventory_list = inventory_str.split('@')
if len(inventory_list) is not 3:
continue
collection, _id, group_name = inventory_list
if collection == 'group':
if _id == 'ungrouped':
group = {
'_id': _id,
}
else:
group = db.collection('groups').find_one(
{'_id': ObjectId(_id)})
if not group:
continue
records = db.collection('machines').find(
{'group': {
'$in': [str(group['_id'])]
}})
for record in records:
hosts[record['hostname']] = {
'ansible_ssh_host': record.get('ansible_ssh_host'),
'ansible_ssh_user': record.get('ansible_ssh_user', 'root'),
'ansible_ssh_port': record.get('ansible_ssh_port', '22'),
}
else:
group_name += '_node'
record = db.collection('machines').find_one({'_id': ObjectId(_id)})
if record:
hosts[record['node_name']] = {
'ansible_ssh_host': record.get('ansible_ssh_host'),
'ansible_ssh_user': record.get('ansible_ssh_user', 'root'),
'ansible_ssh_port': record.get('ansible_ssh_port', 22),
}
if not hosts:
continue
data[group_name] = {'hosts': hosts}
# inventory = json.dumps(hosts)
return data
def add_key():
user = login_user
payload = request.get_json()
if not payload:
return jsonify({
'message': 'illegal params',
'code': 104000,
}), 400
public_key = payload.get('public_key')
name = payload.get('name')
if not public_key:
return jsonify({'message': 'invalid public key', 'code': 104000}), 400
ssh = SSHKey(public_key)
try:
ssh.parse()
except Exception as err:
return jsonify({
'message': 'invalid ssh key: {}'.format(str(err)),
'code': 104001,
}), 400
fingerprint = ssh.hash_md5()
existed = db.collection('public_keys').find_one(
{'fingerprint': fingerprint})
if existed:
return jsonify({
'message': 'ssh public key existed',
'code': 104003
}), 400
options = {'vault_pass': config.vault.get('secret')}
encode = Vault(options).encrypt_string(public_key)
data = {
'fingerprint': fingerprint,
'user_id': user.get('user_id'),
'content': encode,
'name': name,
'created_at': time.time()
}
result = db.collection('public_keys').insert_one(data)
data['_id'] = result.inserted_id
logger.info('add public_keys', extra={'record': data})
return jsonify({
'message': 'ok',
'code': 0,
})
def update_configuration(_id):
payload = request.get_json()
if not payload:
return jsonify({
'message': 'invalid params',
'code': 184000,
}), 400
name = payload.get('name')
record = db.collection('configurations').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040,
}), 404
description = payload.get('description')
maintainer = payload.get('maintainer') or []
variables = payload.get('variables')
status = payload.get('status')
data = {}
if name:
data['name'] = name
if description:
data['description'] = description
if variables:
data['variables'] = variables
if maintainer:
data['maintainer'] = maintainer
if status is not None:
data['status'] = int(status)
if len(data.keys()):
update = {
'$set': data,
}
db.collection('configurations').update_one({'_id': record['_id']},
update=update)
msg = 'update configuration, name: {}'.format(record.get('name'))
logger.info(msg, extra={'record': record, 'changed': data})
return jsonify({
'message': 'ok',
'code': 0,
})
def rollback_one(self, uuid):
record = db.collection('migration').find_one({'uuid': uuid})
if not record or record.get('state') != 'setup':
return False
template = record.get('template')
tasks = template.get('rollback')
self.handler(tasks)
update = {
'$set': {
'state': 'rollback',
'update_at': time.time(),
}
}
db.collection('migration').update_one({'uuid': uuid}, update=update)
def get_permissions(self, user_id, filter=None):
"""
get user permissions
:param user_id: user id
:return: list
"""
user = self.find_by_id(user_id)
if not user:
return []
relate_team = Model.build_model('team_members').find(
{'user_id': user_id})
relate_team = list(relate_team)
team_ids = list(map(lambda i: i.get('team_id'), relate_team))
role_ids = []
menus = []
if team_ids:
team_roles = Model.build_model('team_roles').find(
{'team_id': {
'$in': team_ids
}})
for item in team_roles:
role_ids.append(item.get('role_id'))
roles = db.collection('user_roles').find({'user_id': user_id})
roles = list(roles)
if roles:
ids = map(lambda i: i['role_id'], roles)
role_ids += list(ids)
if role_ids:
where = {'role_id': {'$in': role_ids}}
records = db.collection('role_menus').find(where).sort('id', 1)
for record in records:
where = filter or {}
where['_id'] = ObjectId(record['m_id'])
item = Menu.find_one(where)
if not item or item.get(
'mpid') == '-1' or item.get('status') < 1:
continue
item['actions'] = record.get('actions', ['get'])
menus.append(item)
roles = Role().find_by_ids(role_ids)
return menus, roles
def log_query():
query = request.args or {}
log_type = query.get('logType')
keyword = query.get('keyword')
level = query.get('level')
q = query.get('q')
start = query.get('start')
end = query.get('end')
page = int(query.get('page', 1))
limit = int(query.get('pageSize', 50))
skip = (page - 1) * limit
where = dict()
if log_type:
where['loggerName'] = log_type
if level:
where['level'] = level.upper()
if keyword:
where['message'] = {'$regex': keyword}
if q:
q = dict(parse.parse_qsl(q))
where.update(q)
date = []
if start:
date.append({
'created_at': {
'$gte': int(time.mktime(time.strptime(start, '%Y-%m-%d')))
}
})
if end:
date.append({
'created_at': {
'$lte': int(time.mktime(time.strptime(end, '%Y-%m-%d')))
}
})
if date:
where['$and'] = date
sort = [('_id', -1)]
cursor = db.collection('logs').find(where,
skip=skip,
limit=limit,
sort=sort)
total = cursor.count()
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'list': list(cursor),
'total': total,
'page': page,
'pageSize': limit,
}
})
def get_job(self):
if not self.job_id:
return {}
job = db.collection('jobs').find_one({'_id': ObjectId(self.job_id)})
return job
def verify_mail():
query = request.args
token = query.get('token')
if not query.get('token'):
return jsonify({
'message': 'invalid token',
'code': 104030
}), 403
record = db.collection('mail_verify').find_one({'token': token})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040
}), 404
update = {
'$set': {
'email_status': 1
}
}
User().collection.update_one({'_id': ObjectId(record.get('user_id'))}, update=update)
return jsonify({
'message': 'ok',
'code': 0,
})
def get_current_roles():
username = login_user.get('username')
is_admin = login_user.get('is_admin')
role = Role()
if is_admin:
roles = role.collection.find({})
return jsonify({
'message': 'ok',
'code': 0,
'data': list(roles),
})
user = User()
user_info = user.collection.find_one({'username': username})
where = {
'user_id': str(user_info['_id']),
}
roles = db.collection('user_roles').find(where)
roles = list(roles)
data = []
if roles:
role_ids = map(lambda i: i['role_id'], roles)
data = role.find_by_ids(list(role_ids))
return jsonify({
'message': 'ok',
'code': 0,
'data': data,
})