def test_delete_configuration(self):
configuration = self.get_data('configuration')
playbook = self.get_data('playbook')
configuration['name'] = str(uuid.uuid4())
Configuration.insert_one(configuration)
playbook['register'] = [str(configuration['_id'])]
Playbook.insert_one(playbook)
self.trash += [
[Configuration, configuration['_id']],
[Playbook, playbook['_id']],
]
path = '/configurations/%s' % str(ObjectId())
url = self.get_api_path(path)
response = self.client.delete(url, headers=self.jwt_headers)
self.assert404(response)
self.assertResponseCode(response, 104040)
path = '/configurations/%s' % str(configuration['_id'])
url = self.get_api_path(path)
response = self.client.delete(url, headers=self.jwt_headers)
self.assert403(response)
self.assertResponseCode(response, 104038)
Playbook().collection.delete_one({'_id': playbook['_id']})
path = '/configurations/%s' % str(configuration['_id'])
url = self.get_api_path(path)
response = self.client.delete(url, headers=self.jwt_headers)
self.assert200(response)
Configuration().collection.delete_one({'_id': configuration['_id']})
def check_playbook(book_id):
record = book.find_by_id(book_id)
if not record:
raise Exception('invalid playbook')
records = Playbook.find({'book_id': book_id})
for item in records:
parent = item.get('parent')
if not item.get('parent'):
continue
p_item = Playbook.find_one({'book_id': book_id, 'path': parent})
if not p_item:
p_path = os.path.dirname(parent)
p_path = p_path if p_path != '/' else None
data = {
'path': parent,
'parent': p_path,
'is_dir': True,
'is_edit': False,
'book_id': book_id,
'role': os.path.basename(parent),
'created_at': time.time(),
}
meta = Workspace.get_meta(parent)
data.update(meta)
data['additions'] = meta
Playbook.insert_one(data)
return True
def test_download(self):
url = self.get_api_path('/books/%s/download' % str(ObjectId()))
response = self.client.get(url, headers=self.jwt_headers)
self.assert404(response)
self.assertResponseCode(response, 104040)
data = self.get_data('book')
playbook = self.get_data('playbook')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
playbook_file = self.get_data('playbook_file')
playbook_file['book_id'] = playbook['book_id']
Playbook.insert_one(playbook_file)
self.trash += [[Book, book_id], [Playbook, playbook['_id']],
[Playbook, playbook_file['_id']]]
url = self.get_api_path('/books/%s/download' % str(book_id))
response = self.client.get(url, headers=self.jwt_headers)
# Playbook().collection.delete_one({'_id': playbook['_id']})
# Book().collection.delete_one({'_id': book_id})
self.assert200(response)
headers = response.headers
self.assertEqual(headers['Content-Type'], 'application/zip')
assert len(response.get_data()) > 0
response.close()
def test_delete_book(self):
data = self.get_data('book')
playbook = self.get_data('playbook')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [
[Book, book_id],
[Playbook, playbook['_id']],
]
url = self.get_api_path('/books/' + str(book_id))
not_found_url = self.get_api_path('/books/' + str(ObjectId()))
response = self.client.delete(not_found_url, headers=self.jwt_headers)
self.assert404(response)
self.assertResponseCode(response, 154041)
response = self.client.delete(url, headers=self.jwt_headers)
self.assert200(response)
record = Book.find_by_id(book_id)
self.assertEqual(record, None)
record = Book().collection.find_one({'_id': book_id})
self.assertIsNotNone(record)
self.assertEqual(record['_id'], book_id)
self.assertEqual(record['status'], -1)
playbook_record = Playbook().collection.find_one(
{'_id': playbook['_id']})
self.assertIsNotNone(playbook_record)
self.assertEqual(playbook_record['status'], -1)
def test_rename(self):
data = self.get_data('book')
playbook = self.get_data('playbook')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [
[Book, book_id],
[Playbook, playbook['_id']],
]
path = '/playbook/%s/rename'
url = self.get_api_path(path % str(ObjectId()))
response = self.client.patch(url,
data=self.body({'path': ''}),
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104001)
file_path = os.path.join('newpath', playbook.get('path'))
response = self.client.patch(url,
data=self.body({'path': file_path}),
headers=self.jwt_headers)
self.assert404(response)
self.assertResponseCode(response, 104040)
url = self.get_api_path(path % str(playbook['_id']))
response = self.client.patch(url,
data=self.body({'path': file_path}),
headers=self.jwt_headers)
self.assert200(response)
def test_edit_file(self):
data = self.get_data('book')
playbook = self.get_data('playbook')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [
[Book, book_id],
[Playbook, playbook['_id']],
]
path = '/playbook/%s/file'
url = self.get_api_path(path % str(ObjectId()))
response = self.client.put(url,
data=self.body({}),
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 154000)
url = self.get_api_path(path % str(ObjectId()))
params = playbook.copy()
params.pop('_id')
register_id = str(ObjectId())
update = {
'description': 'big jet plane',
'is_edit': False,
'register': [register_id]
}
params.update(update)
response = self.client.put(url,
data=self.body(params),
headers=self.jwt_headers)
self.assert404(response)
url = self.get_api_path(path % str(playbook['_id']))
with patch('eclogue.api.playbook.Configuration') as config_mock:
config_mock.find.return_value = None
response = self.client.put(url,
data=self.body(params),
headers=self.jwt_headers)
self.assert404(response)
self.assertResponseCode(response, 154042)
config_mock.find.return_value = {'_id': register_id}
with patch('eclogue.api.playbook.Workspace') as wk_mock:
response = self.client.put(url,
data=self.body(params),
headers=self.jwt_headers)
self.assert200(response)
wk_mock.assert_called()
wk_mock.return_value.write_book_file.assert_called()
def delete(_id):
record = db.collection('configurations').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040,
}), 404
is_register = Playbook.find_one({'register': {'$in': [str(_id)]}})
if is_register:
return jsonify({
'message':
'target record is bind to playbook: %s' % is_register.get('name'),
'code':
104038
}), 403
Configuration.delete_one({'_id': record['_id']})
msg = 'update configuration, name: {}'.format(record.get('name'))
logger.info(msg, extra={'record': record, 'force': False})
return jsonify({
'message': 'ok',
'code': 0,
})
def test_upload(self):
url = self.get_api_path('/books/%s/playbook' % str(ObjectId()))
response = self.client.post(url, headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104004)
data = self.get_data('book')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
self.trash += [
[Book, book_id],
]
url = self.get_api_path('/books/%s/playbook' % str(book_id))
response = self.client.post(url, headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104001)
headers = self.jwt_headers.copy()
headers.update({'Content-Type': 'multipart/form-data'})
stream = BytesIO(bytes('mock test', 'utf-8'))
params = {'file': (stream, 'test.yaml')}
response = self.client.post(url,
data=params,
headers=headers,
content_type='multipart/form-data')
self.assert200(response)
record = Playbook.find_one({'book_id': str(book_id)})
assert record is not None
db.fs().delete(record.get('file_id'))
with patch('eclogue.api.book.is_edit') as build_mock:
build_mock.return_value = False
stream = BytesIO(bytes('mock test', 'utf-8'))
params = {'file': (stream, 'binary.mock')}
response = self.client.post(url,
data=params,
headers=headers,
content_type='multipart/form-data')
self.assert200(response)
record = Playbook.find_one({
'book_id': str(book_id),
'file_id': {
'$exists': True
}
})
assert record is not None
db.fs().delete(record.get('file_id'))
Playbook().collection.delete_many({'book_id': str(book_id)})
def test_add_folder(self):
data = self.get_data('book')
playbook = self.get_data('playbook')
playbook['is_dir'] = True
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [
[Book, book_id],
[Playbook, playbook['_id']],
]
path = '/playbook/folder'
url = self.get_api_path(path)
response = self.client.post(url,
data=self.body({
'id': '',
'folder': ''
}),
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104000)
params = {
'id': str(ObjectId()),
'folder': 'test',
'parent': playbook['path'],
'book_id': str(book_id),
}
response = self.client.post(url,
data=self.body(params),
headers=self.jwt_headers)
self.assert400(response)
params['id'] = str(playbook['_id'])
response = self.client.post(url,
data=self.body(params),
headers=self.jwt_headers)
self.assert200(response)
response = self.client.post(url,
data=self.body(params),
headers=self.jwt_headers)
self.assert200(response)
def test_get_playbook(self):
book = self.get_data('book')
playbook = self.get_data('playbook')
result = Book.insert_one(book.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [[Book, book_id], [Playbook, playbook['_id']]]
url = self.get_api_path('/books/%s/playbook' % str(ObjectId()))
response = self.client.get(url, headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 154001)
Book.update_one({'_id': book_id}, {'$set': {'status': -1}})
url = self.get_api_path('/books/%s/playbook' % str(book_id))
response = self.client.get(url, headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 154001)
Book.update_one({'_id': book_id}, {'$set': {'status': 1}})
url = self.get_api_path('/books/%s/playbook' % str(book_id))
response = self.client.get(url, headers=self.jwt_headers)
self.assert200(response)
def test_import_playbook_from_dir(self):
book = self.get_data('book')
self.add_data(Book, book)
book_id = str(book['_id'])
try:
wk = Workspace()
home_path = os.path.dirname(wk.get_book_space(book['name']))
result = wk.import_book_from_dir(home_path,
str(book_id),
prefix='/')
self.assertIs(len(result) > 0, True)
finally:
Playbook().collection.delete_many({'book_id': book_id})
def check_playbook_node(node):
"""
loop check playbook parent node
@param {dict} node
"""
parent_path = os.path.dirname(node['path'])
record = Playbook.find_one({
'book_id': node['book_id'],
'path': parent_path
})
if not record:
current = node.copy()
current['path'] = parent_path
current['parent'] = None if parent_path == '/' else os.path.dirname(
parent_path)
Playbook.insert_one(current)
return check_playbook_node(current)
where = {
'book_id': node['book_id'],
'path': node['path'],
}
return Playbook.update_one(where, node, upsert=True)
def rename(_id):
oid = ObjectId(_id)
body = request.get_json()
file_path = body.get('path')
if not file_path:
return jsonify({
'message': 'invalid param path',
'code': 104001,
}), 400
record = Playbook.find_by_id(oid)
if not record:
return jsonify({
'message': 'record not found',
'code': 104040,
}), 404
if record.get('path') == file_path:
return jsonify({'message': 'ok', 'code': 0, 'data': record})
Playbook().rename(_id, file_path)
return jsonify({'message': 'ok', 'code': 0, 'data': record})
def test_get_playbook(self):
data = self.get_data('book')
playbook = self.get_data('playbook')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [
[Book, book_id],
[Playbook, playbook['_id']],
]
url = self.get_api_path('/books/%s/playbook' % str(ObjectId()))
query = {'current': str(playbook['_id'])}
response = self.client.get(url,
query_string=query,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 154001)
url = self.get_api_path('/books/%s/playbook' % playbook['book_id'])
Book.update_one({'_id': book_id}, {'$set': {'status': -1}})
response = self.client.get(url,
query_string=query,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 154001)
Book.update_one({'_id': book_id}, {'$set': {'status': 1}})
url = self.get_api_path('/books/%s/playbook' % playbook['book_id'])
response = self.client.get(url,
query_string=query,
headers=self.jwt_headers)
self.assert200(response)
result = response.json
data = result.get('data')
check = map(lambda i: str(i['_id']), data)
check = list(check)
assert str(playbook['_id']) in check
def get_register_config(playbook_id):
record = Playbook.find_by_id(playbook_id)
if not record:
return jsonify({
'message': 'record not found',
'code': 104040,
}), 404
records = Configuration().find_by_ids(record.get('register'))
return jsonify({
'message': 'ok',
'code': 0,
'data': records,
})
def add_folder():
params = request.get_json()
if not params or not params.get('id') or not params.get('folder'):
return jsonify({
'message': 'illegal param',
'code': 104000,
}), 400
record_id = params.get('id')
folder = params.get('folder')
parent = params.get('parent')
book_id = params.get('book_id')
parent = parent if parent != '.' else '/'
parent_path = None
if parent != '/':
parent_record = Playbook.find_one({
'_id': ObjectId(record_id),
'is_dir': True
})
if not parent_record:
return jsonify({
'message': 'invalid params',
'code': 104001,
}), 400
parent_path = parent_record.get('path')
file_path = os.path.join(parent, folder)
record = {
'path': file_path,
'book_id': book_id,
'parent': parent_path,
'is_dir': True,
'content': '',
'is_edit': False,
'add_by': login_user.get('username'),
'created_at': int(time.time()),
'updated_at': datetime.datetime.now().isoformat(),
}
meta = get_meta(file_path)
record.update(meta)
record['additions'] = meta
check = Playbook.find_one({'book_id': book_id, 'path': record['path']})
if check:
additions = check.get('additions') or {}
additions.update(meta)
record['additions'] = additions
Playbook.update_one({'_id': check['_id']}, {'$set': record})
else:
Playbook.insert_one(record)
return jsonify({
'message': 'ok',
'code': 0,
})
def get_playbook(_id):
book = Book.find_one({'_id': ObjectId(_id)})
if not book or not int(book.get('status')):
return jsonify({
'message': 'invalid id',
'code': 154001,
}), 400
cursor = Playbook.find({'book_id': str(book.get('_id'))})
cursor = cursor.sort([('is_edit', pymongo.ASCENDING),
('path', pymongo.ASCENDING)])
return jsonify({
'message': 'ok',
'code': 0,
'data': list(cursor),
})
def get_playbook(_id):
book = Book.find_by_id(_id)
if not book or int(book.get('status') == -1):
return jsonify({
'message': 'invalid id',
'code': 154001,
}), 400
cursor = Playbook.find({'book_id': str(book.get('_id'))})
cursor = cursor.sort([('is_edit', pymongo.ASCENDING),
('path', pymongo.ASCENDING)])
# for item in cursor:
# db.collection('playbook').update_one({'_id': item['_id']}, {'$set': {'book_id': str(item.get('book_id'))}})
return jsonify({
'message': 'ok',
'code': 0,
'data': list(cursor),
})
def get_entry(_id):
book = Book.find_one(({'_id': ObjectId(_id)}))
if not book:
return jsonify({'message': 'book not found', 'code': 164000}), 400
where = {
'book_id': str(book.get('_id')),
'is_dir': False,
'role': 'entry',
}
cursor = Playbook.find(where)
return jsonify({
'message': 'ok',
'code': 0,
'data': list(cursor),
})
def install_book_from_dir(dirname, book_id):
wk = Workspace()
bucket = wk.import_book_from_dir(dirname, book_id)
playbooks = Playbook.find({'book_id': book_id})
files = map(lambda i: i['path'], bucket)
paths = map(lambda i: i['path'], playbooks)
diff = list(set(paths) - set(files))
for item in diff:
Playbook.delete_one({'book_id': book_id, 'path': item})
mapping = {}
map(lambda i: {mapping['path']: i}, playbooks)
for item in bucket:
record = mapping.get(item['path'])
if not record:
Playbook.insert_one(item)
continue
if record['additions']:
item['additions'].update(record['additions'])
Playbook.update_one({'_id': record['_id']}, {'$set': item})
def parse_file_inventory(inventory_params):
data = {}
if type(inventory_params) != list:
inventory_params = [inventory_params]
for inventory_str in inventory_params:
inventory_list = inventory_str.split('@')
if len(inventory_list) is not 3:
return False
mark_name, _id, group = inventory_list
record = Playbook.find_one({'_id': ObjectId(_id)})
if not record:
return False
node = parser_inventory(record['content'], True)
# @todo make same as cmdb
# record['content'] = json.dumps(record['content'])
# record['limit'] = group
data.update(node)
return data
def all_books():
query = request.args
job_id = query.get('id')
where = {}
cursor = Book.find({})
records = list(cursor)
for book in records:
def get_children(item):
return {
'value': item['_id'],
'label': item.get('name'),
'isLeaf': True,
}
entries = Playbook.find({'book_id': str(book['_id']), 'role': 'entry'})
children = map(get_children, entries)
book['children'] = list(children)
return jsonify({
'message': 'ok',
'code': 0,
'data': list(records),
})
def install(self, book_id=None):
"""
copy from ansible-galaxy
uses the args list of roles to be installed, unless -f was specified. The list of roles
can be a name (which will be downloaded via the galaxy API and github), or it can be a local .tar.gz file.
"""
role_file = self.options.role_file
if len(self.repo) == 0 and role_file is None:
# the user needs to specify one of either --role-file or specify a single user/role name
raise AnsibleOptionsError("- you must specify a user/role name or a roles file")
no_deps = self.options.no_deps
force = self.options.force
roles_left = []
if role_file:
try:
f = open(role_file, 'r')
if role_file.endswith('.yaml') or role_file.endswith('.yml'):
try:
required_roles = yaml.safe_load(f.read())
except Exception as e:
raise AnsibleError("Unable to load data from the requirements file: %s" % role_file)
if required_roles is None:
raise AnsibleError("No roles found in file: %s" % role_file)
for role in required_roles:
if "include" not in role:
role = RoleRequirement.role_yaml_parse(role)
if "name" not in role and "scm" not in role:
raise AnsibleError("Must specify name or src for role")
roles_left.append(GalaxyRole(self.galaxy, **role))
else:
with open(role["include"]) as f_include:
try:
roles_left += [
GalaxyRole(self.galaxy, **r) for r in
(RoleRequirement.role_yaml_parse(i) for i in yaml.safe_load(f_include))
]
except Exception as e:
msg = "Unable to load data from the include requirements file: %s %s"
raise AnsibleError(msg % (role_file, e))
else:
raise AnsibleError("Invalid role requirements file")
f.close()
except (IOError, OSError) as e:
raise AnsibleError('Unable to open %s: %s' % (role_file, str(e)))
else:
# roles were specified directly, so we'll just go out grab them
# (and their dependencies, unless the user doesn't want us to).
for rname in self.repo:
role = RoleRequirement.role_yaml_parse(rname.strip())
roles_left.append(GalaxyRole(self.galaxy, **role))
installed_role = []
for role in roles_left:
# only process roles in roles files when names matches if given
if role_file and self.repo and role.name not in self.repo:
print('Skipping role %s' % role.name)
continue
# query the galaxy API for the role data
if role.install_info is not None:
if role.install_info['version'] != role.version or force:
if force:
print('- changing role %s from %s to %s' %
(role.name, role.install_info['version'], role.version or "unspecified"))
role.remove()
else:
print('- %s (%s) is already installed - use --force to change version to %s' %
(role.name, role.install_info['version'], role.version or "unspecified"))
installed_role.append(role.name)
continue
else:
if not force:
print('- %s is already installed, skipping.' % str(role))
continue
try:
installed = role.install()
if installed and book_id:
wk = Workspace()
documents = wk.import_book_from_dir(os.path.dirname(role.path), book_id)
for doc in documents:
if doc.get('role') != 'entry':
doc['path'] = '/roles' + doc.get('path')
print(doc['path'])
Playbook().collection.update_one({'path': doc.get('path')}, {'$set': doc}, upsert=True)
except AnsibleError as e:
print("- %s was NOT installed successfully: %s " % (role.name, str(e)))
# self.exit_without_ignore()
continue
# install dependencies, if we want them
if not no_deps and installed:
if not role.metadata:
print("Meta file %s is empty. Skipping dependencies." % role.path)
else:
role_dependencies = role.metadata.get('dependencies') or []
for dep in role_dependencies:
logger.debug('Installing dep %s' % dep)
dep_req = RoleRequirement()
dep_info = dep_req.role_yaml_parse(dep)
dep_role = GalaxyRole(self.galaxy, **dep_info)
if '.' not in dep_role.name and '.' not in dep_role.src and dep_role.scm is None:
# we know we can skip this, as it's not going to
# be found on galaxy.ansible.com
continue
if dep_role.install_info is None:
if dep_role not in roles_left:
print('- adding dependency: %s' % str(dep_role))
roles_left.append(dep_role)
else:
print('- dependency %s already pending installation.' % dep_role.name)
else:
if dep_role.install_info['version'] != dep_role.version:
print(
'- dependency %s from role %s differs from already installed version (%s), skipping' %
(str(dep_role), role.name, dep_role.install_info['version']))
else:
print('- dependency %s is already installed, skipping.' % dep_role.name)
if not installed:
print("- %s was NOT installed successfully." % role.name)
# self.exit_without_ignore()
for role in installed_role:
wk = Workspace()
# wk.import_book_from_dir(self.options.roles_path)
return 0
def upload_playbook(_id):
files = request.files
record = Book.find_by_id(_id)
if not record:
return jsonify({
"message": "book not found",
"code": 104004,
}), 400
if not files:
return jsonify({
'message': 'invalid files params',
'code': 104001
}), 400
file = files['file']
filename = file.filename.lstrip('/')
path_list = filename.split('/')
filename = '/'.join(path_list[1:])
filename = '/' + filename
home_path, basename = os.path.split(filename)
file_list = set(_make_path(filename))
for dirname in file_list:
check = Playbook.find_one({
'book_id': _id,
'path': dirname,
})
if not check:
parent_path, name = os.path.split(dirname)
parent_path = parent_path if parent_path != '/' else None
parent = {
'path': dirname,
'is_dir': True,
'is_edit': False,
'book_id': _id,
'parent': parent_path,
'name': name,
'created_at': time.time(),
}
meta = get_meta(dirname)
parent.update(meta)
parent['additions'] = meta
Playbook.insert_one(parent)
data = {
'path': filename,
'is_dir': False,
'parent': home_path or None,
'book_id': _id
}
can_edit = is_edit(file)
if not can_edit:
file_id = db.save_file(filename=filename, fileobj=file)
data['file_id'] = file_id
else:
content = file.stream.read()
content = content.decode('utf-8')
data['is_encrypt'] = Vault.is_encrypted(content)
if data['is_encrypt']:
# @todo vault password
vault = Vault()
data['content'] = vault.encrypt_string(content)
data['md5'] = md5(content)
else:
data['content'] = content
data['md5'] = md5(content)
meta = get_meta(data['path'])
data.update(meta)
data['additions'] = meta
data['is_edit'] = can_edit
data['created_at'] = time.time()
data['updated_at'] = time.time()
Playbook.update_one({
'path': filename,
'book_id': _id
}, {'$set': data},
upsert=True)
return jsonify({
"message": "ok",
"code": 0,
})
def books():
query = request.args
page = int(query.get('page', 1))
size = int(query.get('pageSize', 50))
offset = (page - 1) * size
keyword = query.get('keyword')
is_admin = login_user.get('is_admin')
start = query.get('start')
end = query.get('end')
maintainer = query.get('maintainer')
where = {'status': {'$ne': -1}}
if keyword:
where['name'] = {'$regex': keyword}
if not is_admin:
where['maintainer'] = {'$in': [login_user.get('username')]}
elif maintainer:
where['maintainer'] = {'$in': [maintainer]}
date = []
if start:
date.append({
'created_at': {
'$gte': int(time.mktime(time.strptime(start, '%Y-%m-%d')))
}
})
if end:
date.append({
'created_at': {
'$lte': int(time.mktime(time.strptime(end, '%Y-%m-%d')))
}
})
if date:
where['$and'] = date
cursor = Book.find(where, skip=offset, limit=size)
total = cursor.count()
records = list(cursor)
data = []
for item in records:
item['job'] = None
where = {
'type': 'playbook',
'template.entry': {
'$in': [str(item['_id'])]
}
}
job = db.collection('jobs').find_one(where)
if job:
item['job'] = {
'_id': job.get('_id'),
'name': job.get('name'),
'type': job.get('type'),
}
if item.get('status'):
data.append(item)
continue
where = {
'role': 'entry',
'book_id': item['_id'],
}
entry = Playbook.find_one(where)
if not entry:
Book.update_one({'_id': item['_id']}, {'$set': {'status': 0}})
item['status'] = 0
data.append(item)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'list': records,
'page': page,
'pageSize': size,
'total': total,
}
})
def job_detail(_id):
query = request.args
record = db.collection('jobs').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040,
}), 404
if record.get('type') == 'adhoc':
template = record.get('template')
inventory = template.get('inventory')
inventory_content = parse_cmdb_inventory(inventory)
template['inventory_content'] = inventory_content
else:
book = db.collection('books').find_one(
{'_id': ObjectId(record.get('book_id'))})
record['book_name'] = book.get('name')
template = record.get('template')
if template:
app_id = template.get('app')
if app_id:
app = db.collection('apps').find_one({'_id': ObjectId(app_id)})
if app:
template['app_name'] = app.get('name')
template['app_params'] = app.get('params')
inventory_type = template.get('inventory_type')
inventory = template.get('inventory')
if inventory_type == 'file':
inventory_content = parse_file_inventory(inventory)
else:
inventory_content = parse_cmdb_inventory(inventory)
template['inventory_content'] = inventory_content
role_ids = template.get('roles')
if role_ids:
roles = Playbook.find_by_ids(role_ids)
template['roles'] = list(map(lambda i: i.get('name'), roles))
page = int(query.get('page', 1))
size = int(query.get('pageSize', 20))
offset = (page - 1) * size
tasks = get_tasks_by_job(_id, offset=offset, limit=size)
tasks = list(tasks)
logs = []
sort = [('_id', -1)]
task = db.collection('tasks').find_one({'job_id': _id}, sort=sort)
if task:
records = db.collection('task_logs').find(
{'task_id': str(task['_id'])})
for item in records:
logs.append({
'content': str(item.get('content')),
'created_at': item.get('created_at')
})
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'job': record,
'logs': logs,
'tasks': tasks,
}
})
def get_job(_id):
username = login_user.get('username')
if not _id:
return jsonify({'message': 'invalid id', 'code': 154000}), 400
job = Job.find_one({
'_id': ObjectId(_id),
'maintainer': {
'$in': [username]
}
})
# @todo job status
if not job:
return jsonify({
'message': 'invalid id',
'code': 154001,
}), 400
template = job.get('template')
inventory_type = template.get('inventory_type')
inventory = template.get('inventory')
if job.get('type') == 'adhoc':
inventory_content = parse_cmdb_inventory(inventory)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
},
})
if inventory_type == 'file':
inventory_content = parse_file_inventory(inventory)
else:
inventory_content = parse_cmdb_inventory(inventory)
check_playbook(job['book_id'])
if inventory_type == 'file':
book = Book.find_one({'_id': ObjectId(job['book_id'])})
if not book:
hosts = []
else:
hosts = get_inventory_by_book(book.get('_id'),
book_name=book.get('name'))
else:
hosts = get_inventory_from_cmdb()
roles = []
condition = {
'book_id': str(job['book_id']),
'role': 'roles',
'is_dir': True
}
parent = Playbook.find_one(condition)
if parent:
where = {
'book_id': job['book_id'],
'is_dir': True,
'parent': parent.get('path')
}
cursor = Playbook.find(where)
roles = list(cursor)
logs = None
task = Task.find_one({'job_id': _id})
if task:
log = db.collection('logs').find_one({'task_id': str(task['_id'])})
if log:
logs = log.get('message')
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
'hosts': hosts,
'roles': roles,
'logs': logs,
},
})
def lint(book_id, options, config=None):
"""
base on ansiblelint
refer to ansiblelint.__main__.py
:param book_id:
:param options:
:param config:
:return: None
"""
formatter = formatters.Formatter()
options = get_default_options(options)
where = {
'book_id': str(book_id),
'role': 'entry',
}
entries = Playbook.find(where)
if not entries:
return False
book = Book.find_by_id(book_id)
if not book:
return False
if config:
if 'quiet' in config:
options.quiet = options.quiet or config['quiet']
if 'parseable' in config:
options.parseable = options.parseable or config['parseable']
if 'parseable_severity' in config:
options.parseable_severity = options.parseable_severity or \
config['parseable_severity']
if 'use_default_rules' in config:
options.use_default_rules = options.use_default_rules or config['use_default_rules']
if 'verbosity' in config:
options.verbosity = options.verbosity + config['verbosity']
options.exclude_paths.extend(
config.get('exclude_paths', []))
if 'rulesdir' in config:
options.rulesdir = options.rulesdir + config['rulesdir']
if 'skip_list' in config:
options.skip_list = options.skip_list + config['skip_list']
if 'tags' in config:
options.tags = options.tags + config['tags']
if options.quiet:
formatter = formatters.QuietFormatter()
if options.parseable:
formatter = formatters.ParseableFormatter()
if options.parseable_severity:
formatter = formatters.ParseableSeverityFormatter()
# no args triggers auto-detection mode
# if len(args) == 0 and not (options.listrules or options.listtags):
# args = get_playbooks_and_roles(options=options)
if options.use_default_rules:
rulesdirs = options.rulesdir + [default_rulesdir]
else:
rulesdirs = options.rulesdir or [default_rulesdir]
rules = RulesCollection()
for rulesdir in rulesdirs:
rules.extend(RulesCollection.create_from_directory(rulesdir))
if options.listrules:
return 0
if options.listtags:
return 0
if isinstance(options.tags, six.string_types):
options.tags = options.tags.split(',')
skip = set()
for s in options.skip_list:
skip.update(str(s).split(','))
options.skip_list = frozenset(skip)
with build_book_from_db(book.get('name'), options.get('roles')) as book_path:
playbooks = []
for record in entries:
entry = os.path.join(book_path, record['path'][1:])
playbooks.append(entry)
playbooks = sorted(set(playbooks))
matches = list()
checked_files = set()
for playbook in playbooks:
runner = Runner(rules, playbook, options.tags,
options.skip_list, options.exclude_paths,
options.verbosity, checked_files)
matches.extend(runner.run())
matches.sort(key=lambda x: (normpath(x.filename), x.linenumber, x.rule.id))
results = []
for match in matches:
filename = str(match.filename)
filename = filename.replace(book_path, '')
results.append({
'lineNumber': match.linenumber,
'line': str(match.line),
'rule': match.rule.id,
'filename': filename,
'message': match.message,
})
return results
def load_ansible_playbook(payload):
template = payload.get('template')
extra = payload.get('extra', {}) or {}
if not template:
return {
'message': 'invalid params',
'code': 104000,
}
name = template.get('name')
if not name:
return {
'message': 'name required',
'code': 104000,
}
entry = template.get('entry')
if len(entry) < 2:
return {
'message': 'entry not found',
'code': 104001,
}
book_id, entry_id = entry
book_record = Book.find_by_id(book_id)
if not book_record:
return {
'message': 'book not found',
'code': 1040011,
}
entry_record = Playbook.find_by_id(entry_id)
if not entry_record:
return {
'message': 'entry not found',
'code': 104001,
}
inventory_type = template.get('inventory_type', 'file')
inventory = template.get('inventory', None)
if not inventory:
return {'message': 'invalid param inventory', 'code': 104002}
if inventory_type == 'file':
inventory_record = parse_file_inventory(inventory)
else:
inventory_record = parse_cmdb_inventory(inventory)
if not inventory_record:
return {'message': 'illegal inventory', 'code': 104002}
group_name = inventory_record.keys()
if not inventory_record:
return {'message': 'invalid param inventory', 'code': 104002}
roles = template.get('roles')
role_names = []
if roles:
roles = list(map(lambda i: ObjectId(i), roles))
check = Playbook.find({
'book_id': book_id,
'_id': {
'$in': roles,
}
})
check = list(check)
if not check:
return {'message': 'invalid param role', 'code': 104003}
role_names = list(map(lambda i: i.get('name'), check))
extra_vars = {
'node': list(group_name),
}
private_key = template.get('private_key')
if private_key:
key_record = Credential.find_one({
'_id': ObjectId(private_key),
'type': 'private_key'
})
if not key_record:
return {'message': 'invalid private key', 'code': 104031}
variables = extra.get('extraVars') or {}
# variables.update({'node': inventory_record.get('limit')})
if type(variables) in [dict, list]:
variables = yaml.safe_dump(variables)
app = template.get('app')
if app:
# @todo status=1
app_record = Application.find_by_id(app)
if not app_record:
return {'message': 'invalid app', 'code': 104043}
app_type = app_record.get('type')
app_params = app_record.get('params')
integration = Integration(app_type, app_params)
check_app = integration.check_app_params()
if not check_app:
return {'message': 'invalid app', 'code': 104014}
job_space = integration.get_job_space(name)
if job_space and variables:
tpl = Template(variables)
variables = tpl.render(ECLOGUE_JOB_SPACE=job_space)
if variables:
variables = yaml.safe_load(variables)
variables and extra_vars.update(variables)
options = dict()
extra_options = template.get('extraOptions')
if extra_options and type(extra_options) == dict:
options = extra_options.copy()
# default_options = get_default_options('playbook')
# for key, value in extra_options.items():
# if default_options.get(key):
# options[key] = value
options['skip_tags'] = template.get('skip_tags', [])
options['inventory'] = inventory_record
# @todo limit
# options['limit'] = inventory_record.get('limit', None)
# options['credential'] = template.get('credential')
options['limit'] = template.get('limit', 'all')
options['forks'] = template.get('forks', 5)
options['tags'] = template.get('tags', [])
options['listtags'] = template.get('listtags', False)
options['listtasks'] = template.get('listtasks', False)
options['timeout'] = template.get('timeout', 10)
options['verbosity'] = template.get('verbosity', 0)
become_method = template.get('become_method')
if become_method:
options['become'] = 'yes'
options['become_method'] = become_method
if template.get('become_user'):
options['become_user'] = template.get('become_user')
vault_pass = template.get('vault_pass')
if vault_pass:
vault_record = Credential.find_one({'_id': ObjectId(vault_pass)})
if not vault_record or not vault_record.get('body'):
return {'message': 'invalid vault pass', 'code': 104004}
vault_body = vault_record.get('body')
vault = Vault({'vault_pass': config.vault.get('secret')})
options['vault_pass'] = vault.decrypt_string(
vault_body.get('vault_pass'))
options['verbosity'] = template.get('verbosity', 0)
options['diff'] = template.get('diff', False)
# options['vault'] = template.get('vault')
options['extra_vars'] = extra_vars
status = int(extra.get('status', 0))
return {
'message': 'ok',
'data': {
'inventory': options['inventory'],
'options': options,
'name': name,
'entry': entry_record['name'],
'book_id': book_id,
'book_name': book_record['name'],
'roles': role_names,
'inventory_type': inventory_type,
'private_key': private_key,
'template': template,
'extra': extra,
'status': status,
}
}
def test_upload(self):
data = self.get_data('book')
playbook = self.get_data('playbook')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [
[Book, book_id],
[Playbook, playbook['_id']],
]
headers = self.jwt_headers.copy()
headers.update({'Content-Type': 'multipart/form-data'})
url = self.get_api_path('/playbook/upload')
response = self.client.post(url, headers=headers)
self.assert400(response)
self.assertResponseCode(response, 104000)
url = self.get_api_path('/playbook/upload')
form = {'parent': '/', 'bookId': str(ObjectId())}
response = self.client.post(url, data=form, headers=headers)
self.assert400(response)
self.assertResponseCode(response, 104001)
stream = BytesIO(bytes('mock test', 'utf-8'))
form['files'] = (stream, 'test.yaml')
response = self.client.post(url, data=form, headers=headers)
self.assert404(response)
self.assertResponseCode(response, 104040)
params = form.copy()
stream = BytesIO(bytes('mock test', 'utf-8'))
params['files'] = (stream, 'test.yaml')
params['parent'] = ObjectId()
response = self.client.post(url,
data=params,
headers=headers,
content_type='multipart/form-data')
self.assert400(response)
self.assertResponseCode(response, 104004)
stream = BytesIO(bytes('mock test', 'utf-8'))
form['files'] = (stream, 'test.yaml')
form['bookId'] = book_id
response = self.client.post(url,
data=form,
headers=headers,
content_type='multipart/form-data')
self.assert200(response)
record = Playbook.find_one({
'book_id': str(book_id),
'_id': {
'$ne': playbook['_id']
}
})
assert record is not None
db.fs().delete(record.get('file_id'))
with patch('eclogue.api.playbook.is_edit') as build_mock:
build_mock.return_value = False
stream = BytesIO(bytes('mock test', 'utf-8'))
form['files'] = (stream, 'binary.mock')
response = self.client.post(url,
data=form,
headers=headers,
content_type='multipart/form-data')
self.assert200(response)
record = Playbook.find_one({
'book_id': str(book_id),
'file_id': {
'$exists': True
}
})
assert record is not None
db.fs().delete(record.get('file_id'))
Playbook().collection.delete_many({'book_id': str(book_id)})
