def get_roles_by_book(_id):
record = Book.find_one(({'_id': ObjectId(_id)}))
if not record:
return jsonify({
'message': 'book not found',
'code': '104001',
}), 400
book_id = str(record['_id'])
check_playbook(book_id)
condition = {'book_id': book_id, 'role': 'roles', 'is_dir': True}
parent = db.collection('playbook').find_one(condition)
if not parent:
return jsonify({
'message': 'ok',
'code': 0,
'data': [],
})
where = {'book_id': book_id, 'is_dir': True, 'parent': parent.get('path')}
cursor = db.collection('playbook').find(where)
records = list(cursor)
return jsonify({
'message': 'ok',
'code': 0,
'data': records,
})
def get_playbook(_id):
book = Book.find_one({'_id': ObjectId(_id)})
if not book or not int(book.get('status')):
return jsonify({
'message': 'invalid id',
'code': 154001,
}), 400
cursor = Playbook.find({'book_id': str(book.get('_id'))})
cursor = cursor.sort([('is_edit', pymongo.ASCENDING),
('path', pymongo.ASCENDING)])
return jsonify({
'message': 'ok',
'code': 0,
'data': list(cursor),
})
def edit_book(_id):
params = request.get_json() or request.form
if not params:
return jsonify({
'message': 'invalid params',
'code': 154000,
}), 400
name = params.get('name')
description = params.get('description')
status = params.get('status', 1)
maintainer = params.get('maintainer', [])
import_type = params.get('importType')
galaxy_repo = params.get('galaxyRepo')
record = Book.find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 154041,
}), 404
data = {
'status': status,
}
if name:
data['name'] = name
if description:
data['description'] = description
if maintainer:
data['maintainer'] = maintainer
if import_type == 'galaxy':
galaxy = AnsibleGalaxy([galaxy_repo], {'force': True})
galaxy.install(record.get('_id'))
Book.update_one({'_id': ObjectId(_id)}, {'$set': data}, upsert=True)
logger.info('book update', extra={'record': record, 'changed': data})
return jsonify({
'message': 'ok',
'code': 0,
'data': data,
})
def get_entry(_id):
book = Book.find_one(({'_id': ObjectId(_id)}))
if not book:
return jsonify({'message': 'book not found', 'code': 164000}), 400
where = {
'book_id': str(book.get('_id')),
'is_dir': False,
'role': 'entry',
}
cursor = Playbook.find(where)
return jsonify({
'message': 'ok',
'code': 0,
'data': list(cursor),
})
def build_book_from_history(self, build_id):
history = db.collection('build_history').find_one(
{'_id': ObjectId(build_id)})
task_id = history.get('task_id')
file_id = history.get('file_id')
job_info = history.get('job_info')
book = Book.find_one({'_id': ObjectId(job_info.get('book_id'))})
bookspace = self.get_book_space(book.get('name'))
bookspace = os.path.join(bookspace, md5(str(task_id)))
self.mkdir(bookspace)
save_file = NamedTemporaryFile(delete=False, suffix='.zip').name
with open(save_file, 'wb') as fd:
db.fs_bucket().download_to_stream(ObjectId(file_id), fd)
extract(save_file, bookspace)
os.unlink(save_file)
return bookspace
def add_book():
params = request.get_json() or request.form
if not params:
return jsonify({
'message': 'invalid params',
'code': 154000,
}), 400
name = params.get('name')
if not name:
return jsonify({
'message': 'name param required',
'code': 154001,
}), 400
existed = Book.find_one({'name': name})
if existed:
return jsonify({
'message': 'book exist',
'code': 154003,
}), 400
description = params.get('description')
status = params.get('status', 1)
bid = params.get('_id')
import_type = params.get('importType')
galaxy_repo = params.get('galaxyRepo')
maintainer = params.get('maintainer', [])
if bid:
record = Book.find_one({'_id': ObjectId(bid)})
if not record:
return jsonify({
'message': 'record not found',
'code': 154041,
}), 404
else:
if import_type == 'galaxy' and galaxy_repo:
galaxy = AnsibleGalaxy([galaxy_repo])
galaxy.install()
logger.info('import galaxy', extra={'repo': galaxy_repo})
data = {
'name': name,
'description': description,
'maintainer': maintainer,
'import_type': import_type,
'galaxy_repo': galaxy_repo,
'status': int(status),
'created_at': int(time.time())
}
result = Book.update_one({'_id': ObjectId(bid)}, {'$set': data},
upsert=True)
data['_id'] = result.upserted_id
return jsonify({
'message': 'ok',
'code': 0,
'data': data,
})
def get_job(_id):
username = login_user.get('username')
if not _id:
return jsonify({'message': 'invalid id', 'code': 154000}), 400
job = Job.find_one({
'_id': ObjectId(_id),
'maintainer': {
'$in': [username]
}
})
# @todo job status
if not job:
return jsonify({
'message': 'invalid id',
'code': 154001,
}), 400
template = job.get('template')
inventory_type = template.get('inventory_type')
inventory = template.get('inventory')
if job.get('type') == 'adhoc':
inventory_content = parse_cmdb_inventory(inventory)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
},
})
if inventory_type == 'file':
inventory_content = parse_file_inventory(inventory)
else:
inventory_content = parse_cmdb_inventory(inventory)
check_playbook(job['book_id'])
if inventory_type == 'file':
book = Book.find_one({'_id': ObjectId(job['book_id'])})
if not book:
hosts = []
else:
hosts = get_inventory_by_book(book.get('_id'),
book_name=book.get('name'))
else:
hosts = get_inventory_from_cmdb()
roles = []
condition = {
'book_id': str(job['book_id']),
'role': 'roles',
'is_dir': True
}
parent = Playbook.find_one(condition)
if parent:
where = {
'book_id': job['book_id'],
'is_dir': True,
'parent': parent.get('path')
}
cursor = Playbook.find(where)
roles = list(cursor)
logs = None
task = Task.find_one({'job_id': _id})
if task:
log = db.collection('logs').find_one({'task_id': str(task['_id'])})
if log:
logs = log.get('message')
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
'hosts': hosts,
'roles': roles,
'logs': logs,
},
})
def load_book_from_db(self, name, roles=None, build_id=False):
book = Book.find_one({'name': name})
if not book:
return False
files = Model.build_model('playbook').find({'book_id': str(book['_id'])})\
.sort([('is_edit', pymongo.ASCENDING), ('path', pymongo.ASCENDING)])
files = list(files)
if not files:
return False
if build_id:
bookspace = os.path.join(self.book, md5(str(build_id)))
else:
bookspace = self.get_book_space(name)
def parse_register(record):
register = record.get('register')
if not register:
return record
c_ids = map(lambda i: ObjectId(i), register)
cfg_records = Configuration.find({'_id': {'$in': list(c_ids)}})
if not cfg_records:
return record
try:
variables = {}
content = yaml.safe_load(record.get('content', ''))
if not content:
return record
vault = Vault({'vault_pass': config.vault.get('secret')})
for cfg in cfg_records:
config_vars = cfg.get('variables')
if not config_vars:
continue
for k, v in config_vars.items():
key = '_'.join(
['ECLOGUE', 'CONFIG',
cfg.get('name', ''), k])
is_encrypt = Vault.is_encrypted(v)
value = v
if is_encrypt:
value = vault.decrypt_string(value)
variables[key] = value
content = dict(content)
content.update(variables)
record['content'] = yaml.safe_dump(content)
except Exception as e:
print(e)
return record
self.check_workspace(path=self._check_make(bookspace))
for item in files:
item = parse_register(item)
if roles and item.get('folder'):
folder = item.get('folder')
if folder and folder not in roles:
continue
filename = bookspace + item.get('path')
if item['is_dir']:
if os.path.isdir(filename):
continue
self.mkdir(filename)
else:
if os.path.isfile(filename):
file_hash = file_md5(filename)
if item.get('md5') and item['md5'] == file_hash:
continue
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
if item['is_edit']:
Model.build_model('playbooks').update_one(
{'_id': item['_id']},
{'$set': {
'md5': md5(item['content'])
}})
with open(filename, 'w') as stream:
stream.write(item['content'])
else:
with open(filename, 'wb') as stream:
db.fs_bucket().download_to_stream(
item['file_id'], stream)
return bookspace
def add_book():
params = request.get_json() or request.form
if not params:
return jsonify({
'message': 'invalid params',
'code': 154000,
}), 400
name = params.get('name')
if not name:
return jsonify({
'message': 'name param required',
'code': 154001,
}), 400
existed = Book.find_one({'name': name})
if existed:
return jsonify({
'message': 'book exist',
'code': 154003,
}), 400
description = params.get('description')
status = params.get('status', 0)
book_id = params.get('_id')
import_type = params.get('importType')
repo = params.get('repo')
maintainer = params.get('maintainer', [])
readonly = params.get('readonly', False)
if book_id:
record = Book.find_by_id(book_id)
if not record:
return jsonify({
'message': 'record not found',
'code': 154041,
}), 404
else:
data = {
'name': name,
'readonly': readonly,
'description': description,
'maintainer': maintainer,
'import_type': import_type,
'repo': repo,
'created_at': int(time.time())
}
result = Book.insert_one(data)
book_id = result.inserted_id
if import_type == 'galaxy' and repo:
galaxy = AnsibleGalaxy([repo])
galaxy.install(book_id)
logger.info('import galaxy', extra={'repo': repo})
elif import_type == 'git' and repo:
git = GitDownload({'repository': repo})
dest = git.install()
Workspace().import_book_from_dir(dest, book_id, exclude=['.git'])
data = {
'readonly': readonly,
'description': description,
'maintainer': maintainer,
'import_type': import_type,
'repo': repo,
'status': int(status),
'updated_at': time.time(),
}
Book.update_one({'_id': ObjectId(book_id)}, {'$set': data}, upsert=True)
data['_id'] = book_id
return jsonify({
'message': 'ok',
'code': 0,
'data': data,
})
def upload():
files = request.files
form = request.form
if not form or not form.get('parent'):
return jsonify({
'message': 'illegal param',
'code': 104000,
}), 400
if not files.get('files'):
return jsonify({
'message': 'illegal param',
'code': 104001,
}), 400
parent_id = form.get('parent')
book_id = form.get('bookId')
if parent_id == '/' and book_id:
book = Book.find_one({'_id': ObjectId(book_id)})
if not book:
return jsonify({
"message": "record not found",
"code": 104040,
}), 404
current = {'path': '/', 'book_id': book_id}
else:
current = Playbook.find_one({'_id': ObjectId(parent_id)})
if not current:
return jsonify({
"message": "current path not found",
"code": 104004,
}), 400
file = files['files']
filename = file.filename
path = os.path.join(current['path'], filename)
# parent = Playbook.find_one({'book_id': book_id, 'path': os.path.dirname(path)})
record = {
'book_id': book_id,
'path': path,
'is_dir': False,
'parent': os.path.dirname(path),
}
meta = get_meta(path)
record.update(meta)
can_edit = is_edit(file)
if not can_edit:
file_id = db.save_file(filename=filename, fileobj=file)
record['file_id'] = file_id
else:
content = file.read()
content = content.decode('utf-8')
record['content'] = content
record['is_edit'] = can_edit
record['created_at'] = int(time.time())
record['updated_at'] = datetime.datetime.now().isoformat()
# where = {
# 'path': path,
# 'book_id': ObjectId(book_id)
# }
# update = {
# '$set': record,
# }
check_playbook_node(record)
# Playbook.update_one(where, update=update, upsert=True)
return jsonify({
'message': 'ok',
'code': 0,
})
def edit_file(_id):
"""
edit playbook file
@todo add lock
:param _id: ObjectId string
:return: json
"""
params = request.get_json() or request.form
if not params:
return jsonify({
'message': 'invalid params',
'code': 154000,
}), 400
edit_type = params.get('type')
if edit_type == 'upload':
return upload_file(_id)
name = params.get('name')
role = params.get('role')
description = params.get('description')
status = params.get('status', 1)
maintainer = params.get('maintainer', [])
can_edit = params.get('is_edit')
is_dir = params.get('is_dir')
is_encrypt = params.get('is_encrypt')
folder = params.get('folder')
register = params.get('register')
content = params.get('content')
record = Playbook.find_by_id(_id)
if not record:
return jsonify({
'message': 'record not found',
'code': 154041,
}), 404
data = {
'status': status,
}
if name:
data['name'] = name
if role:
data['role'] = role
if folder:
data['folder'] = folder
if content:
data['content'] = content
data['md5'] = md5(content)
if description:
data['description'] = description
if maintainer:
data['maintainer'] = maintainer
if can_edit is not None:
data['is_edit'] = bool(is_edit)
data['is_encrypt'] = bool(is_encrypt)
if is_dir is not None:
data['is_dir'] = bool(is_dir)
if register:
c_ids = map(lambda i: ObjectId(i), register)
where = {'_id': {'$in': c_ids}}
register_config = Configuration.find(where)
if not register_config:
return jsonify({
'message': 'invalid register config',
'code': 154042,
}), 404
data['register'] = register
Playbook.update_one({'_id': ObjectId(_id)}, {'$set': data}, upsert=True)
data['_id'] = _id
book = Book.find_one({'_id': ObjectId(record['book_id'])})
wk = Workspace()
wk.write_book_file(book.get('name'), record)
return jsonify({
'message': 'ok',
'code': 0,
'data': data,
})
def import_book_from_dir(self,
home_path,
book_id,
exclude=['*.retry'],
links=False):
bucket = []
cursor = 0
parent = home_path
book_record = Book.find_one({'_id': ObjectId(book_id)})
pattern = '|'.join(exclude).replace('*', '.*?')
for current, dirs, files in os.walk(home_path,
topdown=True,
followlinks=links):
pathname = current.replace(home_path, '') or '/'
if exclude:
match = re.search(pattern, pathname)
if match:
continue
dir_record = {
'book_id': str(book_record.get('_id')),
'path': pathname,
'is_dir': True,
'is_edit': False,
'seq_no': cursor,
'parent': None,
'created_at': int(time.time()),
}
if not current == home_path:
dir_record['parent'] = parent
meta = Workspace.get_meta(pathname=pathname)
dir_record.update(meta)
dir_record['additions'] = meta
parent = pathname
bucket.append(dir_record)
for file in files:
pathname = parent.rstrip('/') + '/' + file
if exclude:
match = re.match(pattern, pathname)
if match:
continue
cursor += 1
filename = current + '/' + file
can_edit = is_edit(filename)
file_record = dir_record.copy()
file_record['is_edit'] = can_edit
file_record['path'] = pathname
file_record['parent'] = parent
file_record['is_dir'] = False
file_record['seq_no'] = cursor
if is_edit:
with open(filename, 'r', encoding='utf-8') as fd:
file_record['content'] = fd.read()
file_record['md5'] = md5(file_record['content'])
file_record['is_encrypt'] = Vault.is_encrypted(
file_record['content'])
meta = self._get_role(file_record['path'])
file_record.update(meta)
file_record['additions'] = meta
bucket.append(file_record)
cursor += 1
is_entry = filter(lambda i: i.get('role') == 'entry', bucket)
is_entry = list(is_entry)
if not is_entry:
path = '/entry.yml'
entry = {
'book_id': str(book_record.get('_id')),
'path': path,
'is_dir': False,
'is_edit': True,
'seq_no': 0,
'content': '',
'parent': None,
'created_at': int(time.time()),
}
meta = self._get_role(path)
entry.update(meta)
entry['additions'] = meta
bucket.append(entry)
return bucket