def test_delete_configuration(self): configuration = self.get_data('configuration') playbook = self.get_data('playbook') configuration['name'] = str(uuid.uuid4()) Configuration.insert_one(configuration) playbook['register'] = [str(configuration['_id'])] Playbook.insert_one(playbook) self.trash += [ [Configuration, configuration['_id']], [Playbook, playbook['_id']], ] path = '/configurations/%s' % str(ObjectId()) url = self.get_api_path(path) response = self.client.delete(url, headers=self.jwt_headers) self.assert404(response) self.assertResponseCode(response, 104040) path = '/configurations/%s' % str(configuration['_id']) url = self.get_api_path(path) response = self.client.delete(url, headers=self.jwt_headers) self.assert403(response) self.assertResponseCode(response, 104038) Playbook().collection.delete_one({'_id': playbook['_id']}) path = '/configurations/%s' % str(configuration['_id']) url = self.get_api_path(path) response = self.client.delete(url, headers=self.jwt_headers) self.assert200(response) Configuration().collection.delete_one({'_id': configuration['_id']})
def check_playbook(book_id): record = book.find_by_id(book_id) if not record: raise Exception('invalid playbook') records = Playbook.find({'book_id': book_id}) for item in records: parent = item.get('parent') if not item.get('parent'): continue p_item = Playbook.find_one({'book_id': book_id, 'path': parent}) if not p_item: p_path = os.path.dirname(parent) p_path = p_path if p_path != '/' else None data = { 'path': parent, 'parent': p_path, 'is_dir': True, 'is_edit': False, 'book_id': book_id, 'role': os.path.basename(parent), 'created_at': time.time(), } meta = Workspace.get_meta(parent) data.update(meta) data['additions'] = meta Playbook.insert_one(data) return True
def test_download(self): url = self.get_api_path('/books/%s/download' % str(ObjectId())) response = self.client.get(url, headers=self.jwt_headers) self.assert404(response) self.assertResponseCode(response, 104040) data = self.get_data('book') playbook = self.get_data('playbook') data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) playbook_file = self.get_data('playbook_file') playbook_file['book_id'] = playbook['book_id'] Playbook.insert_one(playbook_file) self.trash += [[Book, book_id], [Playbook, playbook['_id']], [Playbook, playbook_file['_id']]] url = self.get_api_path('/books/%s/download' % str(book_id)) response = self.client.get(url, headers=self.jwt_headers) # Playbook().collection.delete_one({'_id': playbook['_id']}) # Book().collection.delete_one({'_id': book_id}) self.assert200(response) headers = response.headers self.assertEqual(headers['Content-Type'], 'application/zip') assert len(response.get_data()) > 0 response.close()
def test_delete_book(self): data = self.get_data('book') playbook = self.get_data('playbook') data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) self.trash += [ [Book, book_id], [Playbook, playbook['_id']], ] url = self.get_api_path('/books/' + str(book_id)) not_found_url = self.get_api_path('/books/' + str(ObjectId())) response = self.client.delete(not_found_url, headers=self.jwt_headers) self.assert404(response) self.assertResponseCode(response, 154041) response = self.client.delete(url, headers=self.jwt_headers) self.assert200(response) record = Book.find_by_id(book_id) self.assertEqual(record, None) record = Book().collection.find_one({'_id': book_id}) self.assertIsNotNone(record) self.assertEqual(record['_id'], book_id) self.assertEqual(record['status'], -1) playbook_record = Playbook().collection.find_one( {'_id': playbook['_id']}) self.assertIsNotNone(playbook_record) self.assertEqual(playbook_record['status'], -1)
def test_rename(self): data = self.get_data('book') playbook = self.get_data('playbook') data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) self.trash += [ [Book, book_id], [Playbook, playbook['_id']], ] path = '/playbook/%s/rename' url = self.get_api_path(path % str(ObjectId())) response = self.client.patch(url, data=self.body({'path': ''}), headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 104001) file_path = os.path.join('newpath', playbook.get('path')) response = self.client.patch(url, data=self.body({'path': file_path}), headers=self.jwt_headers) self.assert404(response) self.assertResponseCode(response, 104040) url = self.get_api_path(path % str(playbook['_id'])) response = self.client.patch(url, data=self.body({'path': file_path}), headers=self.jwt_headers) self.assert200(response)
def test_edit_file(self): data = self.get_data('book') playbook = self.get_data('playbook') data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) self.trash += [ [Book, book_id], [Playbook, playbook['_id']], ] path = '/playbook/%s/file' url = self.get_api_path(path % str(ObjectId())) response = self.client.put(url, data=self.body({}), headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 154000) url = self.get_api_path(path % str(ObjectId())) params = playbook.copy() params.pop('_id') register_id = str(ObjectId()) update = { 'description': 'big jet plane', 'is_edit': False, 'register': [register_id] } params.update(update) response = self.client.put(url, data=self.body(params), headers=self.jwt_headers) self.assert404(response) url = self.get_api_path(path % str(playbook['_id'])) with patch('eclogue.api.playbook.Configuration') as config_mock: config_mock.find.return_value = None response = self.client.put(url, data=self.body(params), headers=self.jwt_headers) self.assert404(response) self.assertResponseCode(response, 154042) config_mock.find.return_value = {'_id': register_id} with patch('eclogue.api.playbook.Workspace') as wk_mock: response = self.client.put(url, data=self.body(params), headers=self.jwt_headers) self.assert200(response) wk_mock.assert_called() wk_mock.return_value.write_book_file.assert_called()
def delete(_id): record = db.collection('configurations').find_one({'_id': ObjectId(_id)}) if not record: return jsonify({ 'message': 'record not found', 'code': 104040, }), 404 is_register = Playbook.find_one({'register': {'$in': [str(_id)]}}) if is_register: return jsonify({ 'message': 'target record is bind to playbook: %s' % is_register.get('name'), 'code': 104038 }), 403 Configuration.delete_one({'_id': record['_id']}) msg = 'update configuration, name: {}'.format(record.get('name')) logger.info(msg, extra={'record': record, 'force': False}) return jsonify({ 'message': 'ok', 'code': 0, })
def test_upload(self): url = self.get_api_path('/books/%s/playbook' % str(ObjectId())) response = self.client.post(url, headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 104004) data = self.get_data('book') data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id self.trash += [ [Book, book_id], ] url = self.get_api_path('/books/%s/playbook' % str(book_id)) response = self.client.post(url, headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 104001) headers = self.jwt_headers.copy() headers.update({'Content-Type': 'multipart/form-data'}) stream = BytesIO(bytes('mock test', 'utf-8')) params = {'file': (stream, 'test.yaml')} response = self.client.post(url, data=params, headers=headers, content_type='multipart/form-data') self.assert200(response) record = Playbook.find_one({'book_id': str(book_id)}) assert record is not None db.fs().delete(record.get('file_id')) with patch('eclogue.api.book.is_edit') as build_mock: build_mock.return_value = False stream = BytesIO(bytes('mock test', 'utf-8')) params = {'file': (stream, 'binary.mock')} response = self.client.post(url, data=params, headers=headers, content_type='multipart/form-data') self.assert200(response) record = Playbook.find_one({ 'book_id': str(book_id), 'file_id': { '$exists': True } }) assert record is not None db.fs().delete(record.get('file_id')) Playbook().collection.delete_many({'book_id': str(book_id)})
def test_add_folder(self): data = self.get_data('book') playbook = self.get_data('playbook') playbook['is_dir'] = True data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) self.trash += [ [Book, book_id], [Playbook, playbook['_id']], ] path = '/playbook/folder' url = self.get_api_path(path) response = self.client.post(url, data=self.body({ 'id': '', 'folder': '' }), headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 104000) params = { 'id': str(ObjectId()), 'folder': 'test', 'parent': playbook['path'], 'book_id': str(book_id), } response = self.client.post(url, data=self.body(params), headers=self.jwt_headers) self.assert400(response) params['id'] = str(playbook['_id']) response = self.client.post(url, data=self.body(params), headers=self.jwt_headers) self.assert200(response) response = self.client.post(url, data=self.body(params), headers=self.jwt_headers) self.assert200(response)
def test_get_playbook(self): book = self.get_data('book') playbook = self.get_data('playbook') result = Book.insert_one(book.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) self.trash += [[Book, book_id], [Playbook, playbook['_id']]] url = self.get_api_path('/books/%s/playbook' % str(ObjectId())) response = self.client.get(url, headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 154001) Book.update_one({'_id': book_id}, {'$set': {'status': -1}}) url = self.get_api_path('/books/%s/playbook' % str(book_id)) response = self.client.get(url, headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 154001) Book.update_one({'_id': book_id}, {'$set': {'status': 1}}) url = self.get_api_path('/books/%s/playbook' % str(book_id)) response = self.client.get(url, headers=self.jwt_headers) self.assert200(response)
def test_import_playbook_from_dir(self): book = self.get_data('book') self.add_data(Book, book) book_id = str(book['_id']) try: wk = Workspace() home_path = os.path.dirname(wk.get_book_space(book['name'])) result = wk.import_book_from_dir(home_path, str(book_id), prefix='/') self.assertIs(len(result) > 0, True) finally: Playbook().collection.delete_many({'book_id': book_id})
def check_playbook_node(node): """ loop check playbook parent node @param {dict} node """ parent_path = os.path.dirname(node['path']) record = Playbook.find_one({ 'book_id': node['book_id'], 'path': parent_path }) if not record: current = node.copy() current['path'] = parent_path current['parent'] = None if parent_path == '/' else os.path.dirname( parent_path) Playbook.insert_one(current) return check_playbook_node(current) where = { 'book_id': node['book_id'], 'path': node['path'], } return Playbook.update_one(where, node, upsert=True)
def rename(_id): oid = ObjectId(_id) body = request.get_json() file_path = body.get('path') if not file_path: return jsonify({ 'message': 'invalid param path', 'code': 104001, }), 400 record = Playbook.find_by_id(oid) if not record: return jsonify({ 'message': 'record not found', 'code': 104040, }), 404 if record.get('path') == file_path: return jsonify({'message': 'ok', 'code': 0, 'data': record}) Playbook().rename(_id, file_path) return jsonify({'message': 'ok', 'code': 0, 'data': record})
def test_get_playbook(self): data = self.get_data('book') playbook = self.get_data('playbook') data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) self.trash += [ [Book, book_id], [Playbook, playbook['_id']], ] url = self.get_api_path('/books/%s/playbook' % str(ObjectId())) query = {'current': str(playbook['_id'])} response = self.client.get(url, query_string=query, headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 154001) url = self.get_api_path('/books/%s/playbook' % playbook['book_id']) Book.update_one({'_id': book_id}, {'$set': {'status': -1}}) response = self.client.get(url, query_string=query, headers=self.jwt_headers) self.assert400(response) self.assertResponseCode(response, 154001) Book.update_one({'_id': book_id}, {'$set': {'status': 1}}) url = self.get_api_path('/books/%s/playbook' % playbook['book_id']) response = self.client.get(url, query_string=query, headers=self.jwt_headers) self.assert200(response) result = response.json data = result.get('data') check = map(lambda i: str(i['_id']), data) check = list(check) assert str(playbook['_id']) in check
def get_register_config(playbook_id): record = Playbook.find_by_id(playbook_id) if not record: return jsonify({ 'message': 'record not found', 'code': 104040, }), 404 records = Configuration().find_by_ids(record.get('register')) return jsonify({ 'message': 'ok', 'code': 0, 'data': records, })
def add_folder(): params = request.get_json() if not params or not params.get('id') or not params.get('folder'): return jsonify({ 'message': 'illegal param', 'code': 104000, }), 400 record_id = params.get('id') folder = params.get('folder') parent = params.get('parent') book_id = params.get('book_id') parent = parent if parent != '.' else '/' parent_path = None if parent != '/': parent_record = Playbook.find_one({ '_id': ObjectId(record_id), 'is_dir': True }) if not parent_record: return jsonify({ 'message': 'invalid params', 'code': 104001, }), 400 parent_path = parent_record.get('path') file_path = os.path.join(parent, folder) record = { 'path': file_path, 'book_id': book_id, 'parent': parent_path, 'is_dir': True, 'content': '', 'is_edit': False, 'add_by': login_user.get('username'), 'created_at': int(time.time()), 'updated_at': datetime.datetime.now().isoformat(), } meta = get_meta(file_path) record.update(meta) record['additions'] = meta check = Playbook.find_one({'book_id': book_id, 'path': record['path']}) if check: additions = check.get('additions') or {} additions.update(meta) record['additions'] = additions Playbook.update_one({'_id': check['_id']}, {'$set': record}) else: Playbook.insert_one(record) return jsonify({ 'message': 'ok', 'code': 0, })
def get_playbook(_id): book = Book.find_one({'_id': ObjectId(_id)}) if not book or not int(book.get('status')): return jsonify({ 'message': 'invalid id', 'code': 154001, }), 400 cursor = Playbook.find({'book_id': str(book.get('_id'))}) cursor = cursor.sort([('is_edit', pymongo.ASCENDING), ('path', pymongo.ASCENDING)]) return jsonify({ 'message': 'ok', 'code': 0, 'data': list(cursor), })
def get_playbook(_id): book = Book.find_by_id(_id) if not book or int(book.get('status') == -1): return jsonify({ 'message': 'invalid id', 'code': 154001, }), 400 cursor = Playbook.find({'book_id': str(book.get('_id'))}) cursor = cursor.sort([('is_edit', pymongo.ASCENDING), ('path', pymongo.ASCENDING)]) # for item in cursor: # db.collection('playbook').update_one({'_id': item['_id']}, {'$set': {'book_id': str(item.get('book_id'))}}) return jsonify({ 'message': 'ok', 'code': 0, 'data': list(cursor), })
def get_entry(_id): book = Book.find_one(({'_id': ObjectId(_id)})) if not book: return jsonify({'message': 'book not found', 'code': 164000}), 400 where = { 'book_id': str(book.get('_id')), 'is_dir': False, 'role': 'entry', } cursor = Playbook.find(where) return jsonify({ 'message': 'ok', 'code': 0, 'data': list(cursor), })
def install_book_from_dir(dirname, book_id): wk = Workspace() bucket = wk.import_book_from_dir(dirname, book_id) playbooks = Playbook.find({'book_id': book_id}) files = map(lambda i: i['path'], bucket) paths = map(lambda i: i['path'], playbooks) diff = list(set(paths) - set(files)) for item in diff: Playbook.delete_one({'book_id': book_id, 'path': item}) mapping = {} map(lambda i: {mapping['path']: i}, playbooks) for item in bucket: record = mapping.get(item['path']) if not record: Playbook.insert_one(item) continue if record['additions']: item['additions'].update(record['additions']) Playbook.update_one({'_id': record['_id']}, {'$set': item})
def parse_file_inventory(inventory_params): data = {} if type(inventory_params) != list: inventory_params = [inventory_params] for inventory_str in inventory_params: inventory_list = inventory_str.split('@') if len(inventory_list) is not 3: return False mark_name, _id, group = inventory_list record = Playbook.find_one({'_id': ObjectId(_id)}) if not record: return False node = parser_inventory(record['content'], True) # @todo make same as cmdb # record['content'] = json.dumps(record['content']) # record['limit'] = group data.update(node) return data
def all_books(): query = request.args job_id = query.get('id') where = {} cursor = Book.find({}) records = list(cursor) for book in records: def get_children(item): return { 'value': item['_id'], 'label': item.get('name'), 'isLeaf': True, } entries = Playbook.find({'book_id': str(book['_id']), 'role': 'entry'}) children = map(get_children, entries) book['children'] = list(children) return jsonify({ 'message': 'ok', 'code': 0, 'data': list(records), })
def install(self, book_id=None): """ copy from ansible-galaxy uses the args list of roles to be installed, unless -f was specified. The list of roles can be a name (which will be downloaded via the galaxy API and github), or it can be a local .tar.gz file. """ role_file = self.options.role_file if len(self.repo) == 0 and role_file is None: # the user needs to specify one of either --role-file or specify a single user/role name raise AnsibleOptionsError("- you must specify a user/role name or a roles file") no_deps = self.options.no_deps force = self.options.force roles_left = [] if role_file: try: f = open(role_file, 'r') if role_file.endswith('.yaml') or role_file.endswith('.yml'): try: required_roles = yaml.safe_load(f.read()) except Exception as e: raise AnsibleError("Unable to load data from the requirements file: %s" % role_file) if required_roles is None: raise AnsibleError("No roles found in file: %s" % role_file) for role in required_roles: if "include" not in role: role = RoleRequirement.role_yaml_parse(role) if "name" not in role and "scm" not in role: raise AnsibleError("Must specify name or src for role") roles_left.append(GalaxyRole(self.galaxy, **role)) else: with open(role["include"]) as f_include: try: roles_left += [ GalaxyRole(self.galaxy, **r) for r in (RoleRequirement.role_yaml_parse(i) for i in yaml.safe_load(f_include)) ] except Exception as e: msg = "Unable to load data from the include requirements file: %s %s" raise AnsibleError(msg % (role_file, e)) else: raise AnsibleError("Invalid role requirements file") f.close() except (IOError, OSError) as e: raise AnsibleError('Unable to open %s: %s' % (role_file, str(e))) else: # roles were specified directly, so we'll just go out grab them # (and their dependencies, unless the user doesn't want us to). for rname in self.repo: role = RoleRequirement.role_yaml_parse(rname.strip()) roles_left.append(GalaxyRole(self.galaxy, **role)) installed_role = [] for role in roles_left: # only process roles in roles files when names matches if given if role_file and self.repo and role.name not in self.repo: print('Skipping role %s' % role.name) continue # query the galaxy API for the role data if role.install_info is not None: if role.install_info['version'] != role.version or force: if force: print('- changing role %s from %s to %s' % (role.name, role.install_info['version'], role.version or "unspecified")) role.remove() else: print('- %s (%s) is already installed - use --force to change version to %s' % (role.name, role.install_info['version'], role.version or "unspecified")) installed_role.append(role.name) continue else: if not force: print('- %s is already installed, skipping.' % str(role)) continue try: installed = role.install() if installed and book_id: wk = Workspace() documents = wk.import_book_from_dir(os.path.dirname(role.path), book_id) for doc in documents: if doc.get('role') != 'entry': doc['path'] = '/roles' + doc.get('path') print(doc['path']) Playbook().collection.update_one({'path': doc.get('path')}, {'$set': doc}, upsert=True) except AnsibleError as e: print("- %s was NOT installed successfully: %s " % (role.name, str(e))) # self.exit_without_ignore() continue # install dependencies, if we want them if not no_deps and installed: if not role.metadata: print("Meta file %s is empty. Skipping dependencies." % role.path) else: role_dependencies = role.metadata.get('dependencies') or [] for dep in role_dependencies: logger.debug('Installing dep %s' % dep) dep_req = RoleRequirement() dep_info = dep_req.role_yaml_parse(dep) dep_role = GalaxyRole(self.galaxy, **dep_info) if '.' not in dep_role.name and '.' not in dep_role.src and dep_role.scm is None: # we know we can skip this, as it's not going to # be found on galaxy.ansible.com continue if dep_role.install_info is None: if dep_role not in roles_left: print('- adding dependency: %s' % str(dep_role)) roles_left.append(dep_role) else: print('- dependency %s already pending installation.' % dep_role.name) else: if dep_role.install_info['version'] != dep_role.version: print( '- dependency %s from role %s differs from already installed version (%s), skipping' % (str(dep_role), role.name, dep_role.install_info['version'])) else: print('- dependency %s is already installed, skipping.' % dep_role.name) if not installed: print("- %s was NOT installed successfully." % role.name) # self.exit_without_ignore() for role in installed_role: wk = Workspace() # wk.import_book_from_dir(self.options.roles_path) return 0
def upload_playbook(_id): files = request.files record = Book.find_by_id(_id) if not record: return jsonify({ "message": "book not found", "code": 104004, }), 400 if not files: return jsonify({ 'message': 'invalid files params', 'code': 104001 }), 400 file = files['file'] filename = file.filename.lstrip('/') path_list = filename.split('/') filename = '/'.join(path_list[1:]) filename = '/' + filename home_path, basename = os.path.split(filename) file_list = set(_make_path(filename)) for dirname in file_list: check = Playbook.find_one({ 'book_id': _id, 'path': dirname, }) if not check: parent_path, name = os.path.split(dirname) parent_path = parent_path if parent_path != '/' else None parent = { 'path': dirname, 'is_dir': True, 'is_edit': False, 'book_id': _id, 'parent': parent_path, 'name': name, 'created_at': time.time(), } meta = get_meta(dirname) parent.update(meta) parent['additions'] = meta Playbook.insert_one(parent) data = { 'path': filename, 'is_dir': False, 'parent': home_path or None, 'book_id': _id } can_edit = is_edit(file) if not can_edit: file_id = db.save_file(filename=filename, fileobj=file) data['file_id'] = file_id else: content = file.stream.read() content = content.decode('utf-8') data['is_encrypt'] = Vault.is_encrypted(content) if data['is_encrypt']: # @todo vault password vault = Vault() data['content'] = vault.encrypt_string(content) data['md5'] = md5(content) else: data['content'] = content data['md5'] = md5(content) meta = get_meta(data['path']) data.update(meta) data['additions'] = meta data['is_edit'] = can_edit data['created_at'] = time.time() data['updated_at'] = time.time() Playbook.update_one({ 'path': filename, 'book_id': _id }, {'$set': data}, upsert=True) return jsonify({ "message": "ok", "code": 0, })
def books(): query = request.args page = int(query.get('page', 1)) size = int(query.get('pageSize', 50)) offset = (page - 1) * size keyword = query.get('keyword') is_admin = login_user.get('is_admin') start = query.get('start') end = query.get('end') maintainer = query.get('maintainer') where = {'status': {'$ne': -1}} if keyword: where['name'] = {'$regex': keyword} if not is_admin: where['maintainer'] = {'$in': [login_user.get('username')]} elif maintainer: where['maintainer'] = {'$in': [maintainer]} date = [] if start: date.append({ 'created_at': { '$gte': int(time.mktime(time.strptime(start, '%Y-%m-%d'))) } }) if end: date.append({ 'created_at': { '$lte': int(time.mktime(time.strptime(end, '%Y-%m-%d'))) } }) if date: where['$and'] = date cursor = Book.find(where, skip=offset, limit=size) total = cursor.count() records = list(cursor) data = [] for item in records: item['job'] = None where = { 'type': 'playbook', 'template.entry': { '$in': [str(item['_id'])] } } job = db.collection('jobs').find_one(where) if job: item['job'] = { '_id': job.get('_id'), 'name': job.get('name'), 'type': job.get('type'), } if item.get('status'): data.append(item) continue where = { 'role': 'entry', 'book_id': item['_id'], } entry = Playbook.find_one(where) if not entry: Book.update_one({'_id': item['_id']}, {'$set': {'status': 0}}) item['status'] = 0 data.append(item) return jsonify({ 'message': 'ok', 'code': 0, 'data': { 'list': records, 'page': page, 'pageSize': size, 'total': total, } })
def job_detail(_id): query = request.args record = db.collection('jobs').find_one({'_id': ObjectId(_id)}) if not record: return jsonify({ 'message': 'record not found', 'code': 104040, }), 404 if record.get('type') == 'adhoc': template = record.get('template') inventory = template.get('inventory') inventory_content = parse_cmdb_inventory(inventory) template['inventory_content'] = inventory_content else: book = db.collection('books').find_one( {'_id': ObjectId(record.get('book_id'))}) record['book_name'] = book.get('name') template = record.get('template') if template: app_id = template.get('app') if app_id: app = db.collection('apps').find_one({'_id': ObjectId(app_id)}) if app: template['app_name'] = app.get('name') template['app_params'] = app.get('params') inventory_type = template.get('inventory_type') inventory = template.get('inventory') if inventory_type == 'file': inventory_content = parse_file_inventory(inventory) else: inventory_content = parse_cmdb_inventory(inventory) template['inventory_content'] = inventory_content role_ids = template.get('roles') if role_ids: roles = Playbook.find_by_ids(role_ids) template['roles'] = list(map(lambda i: i.get('name'), roles)) page = int(query.get('page', 1)) size = int(query.get('pageSize', 20)) offset = (page - 1) * size tasks = get_tasks_by_job(_id, offset=offset, limit=size) tasks = list(tasks) logs = [] sort = [('_id', -1)] task = db.collection('tasks').find_one({'job_id': _id}, sort=sort) if task: records = db.collection('task_logs').find( {'task_id': str(task['_id'])}) for item in records: logs.append({ 'content': str(item.get('content')), 'created_at': item.get('created_at') }) return jsonify({ 'message': 'ok', 'code': 0, 'data': { 'job': record, 'logs': logs, 'tasks': tasks, } })
def get_job(_id): username = login_user.get('username') if not _id: return jsonify({'message': 'invalid id', 'code': 154000}), 400 job = Job.find_one({ '_id': ObjectId(_id), 'maintainer': { '$in': [username] } }) # @todo job status if not job: return jsonify({ 'message': 'invalid id', 'code': 154001, }), 400 template = job.get('template') inventory_type = template.get('inventory_type') inventory = template.get('inventory') if job.get('type') == 'adhoc': inventory_content = parse_cmdb_inventory(inventory) return jsonify({ 'message': 'ok', 'code': 0, 'data': { 'record': job, 'previewContent': inventory_content, }, }) if inventory_type == 'file': inventory_content = parse_file_inventory(inventory) else: inventory_content = parse_cmdb_inventory(inventory) check_playbook(job['book_id']) if inventory_type == 'file': book = Book.find_one({'_id': ObjectId(job['book_id'])}) if not book: hosts = [] else: hosts = get_inventory_by_book(book.get('_id'), book_name=book.get('name')) else: hosts = get_inventory_from_cmdb() roles = [] condition = { 'book_id': str(job['book_id']), 'role': 'roles', 'is_dir': True } parent = Playbook.find_one(condition) if parent: where = { 'book_id': job['book_id'], 'is_dir': True, 'parent': parent.get('path') } cursor = Playbook.find(where) roles = list(cursor) logs = None task = Task.find_one({'job_id': _id}) if task: log = db.collection('logs').find_one({'task_id': str(task['_id'])}) if log: logs = log.get('message') return jsonify({ 'message': 'ok', 'code': 0, 'data': { 'record': job, 'previewContent': inventory_content, 'hosts': hosts, 'roles': roles, 'logs': logs, }, })
def lint(book_id, options, config=None): """ base on ansiblelint refer to ansiblelint.__main__.py :param book_id: :param options: :param config: :return: None """ formatter = formatters.Formatter() options = get_default_options(options) where = { 'book_id': str(book_id), 'role': 'entry', } entries = Playbook.find(where) if not entries: return False book = Book.find_by_id(book_id) if not book: return False if config: if 'quiet' in config: options.quiet = options.quiet or config['quiet'] if 'parseable' in config: options.parseable = options.parseable or config['parseable'] if 'parseable_severity' in config: options.parseable_severity = options.parseable_severity or \ config['parseable_severity'] if 'use_default_rules' in config: options.use_default_rules = options.use_default_rules or config['use_default_rules'] if 'verbosity' in config: options.verbosity = options.verbosity + config['verbosity'] options.exclude_paths.extend( config.get('exclude_paths', [])) if 'rulesdir' in config: options.rulesdir = options.rulesdir + config['rulesdir'] if 'skip_list' in config: options.skip_list = options.skip_list + config['skip_list'] if 'tags' in config: options.tags = options.tags + config['tags'] if options.quiet: formatter = formatters.QuietFormatter() if options.parseable: formatter = formatters.ParseableFormatter() if options.parseable_severity: formatter = formatters.ParseableSeverityFormatter() # no args triggers auto-detection mode # if len(args) == 0 and not (options.listrules or options.listtags): # args = get_playbooks_and_roles(options=options) if options.use_default_rules: rulesdirs = options.rulesdir + [default_rulesdir] else: rulesdirs = options.rulesdir or [default_rulesdir] rules = RulesCollection() for rulesdir in rulesdirs: rules.extend(RulesCollection.create_from_directory(rulesdir)) if options.listrules: return 0 if options.listtags: return 0 if isinstance(options.tags, six.string_types): options.tags = options.tags.split(',') skip = set() for s in options.skip_list: skip.update(str(s).split(',')) options.skip_list = frozenset(skip) with build_book_from_db(book.get('name'), options.get('roles')) as book_path: playbooks = [] for record in entries: entry = os.path.join(book_path, record['path'][1:]) playbooks.append(entry) playbooks = sorted(set(playbooks)) matches = list() checked_files = set() for playbook in playbooks: runner = Runner(rules, playbook, options.tags, options.skip_list, options.exclude_paths, options.verbosity, checked_files) matches.extend(runner.run()) matches.sort(key=lambda x: (normpath(x.filename), x.linenumber, x.rule.id)) results = [] for match in matches: filename = str(match.filename) filename = filename.replace(book_path, '') results.append({ 'lineNumber': match.linenumber, 'line': str(match.line), 'rule': match.rule.id, 'filename': filename, 'message': match.message, }) return results
def load_ansible_playbook(payload): template = payload.get('template') extra = payload.get('extra', {}) or {} if not template: return { 'message': 'invalid params', 'code': 104000, } name = template.get('name') if not name: return { 'message': 'name required', 'code': 104000, } entry = template.get('entry') if len(entry) < 2: return { 'message': 'entry not found', 'code': 104001, } book_id, entry_id = entry book_record = Book.find_by_id(book_id) if not book_record: return { 'message': 'book not found', 'code': 1040011, } entry_record = Playbook.find_by_id(entry_id) if not entry_record: return { 'message': 'entry not found', 'code': 104001, } inventory_type = template.get('inventory_type', 'file') inventory = template.get('inventory', None) if not inventory: return {'message': 'invalid param inventory', 'code': 104002} if inventory_type == 'file': inventory_record = parse_file_inventory(inventory) else: inventory_record = parse_cmdb_inventory(inventory) if not inventory_record: return {'message': 'illegal inventory', 'code': 104002} group_name = inventory_record.keys() if not inventory_record: return {'message': 'invalid param inventory', 'code': 104002} roles = template.get('roles') role_names = [] if roles: roles = list(map(lambda i: ObjectId(i), roles)) check = Playbook.find({ 'book_id': book_id, '_id': { '$in': roles, } }) check = list(check) if not check: return {'message': 'invalid param role', 'code': 104003} role_names = list(map(lambda i: i.get('name'), check)) extra_vars = { 'node': list(group_name), } private_key = template.get('private_key') if private_key: key_record = Credential.find_one({ '_id': ObjectId(private_key), 'type': 'private_key' }) if not key_record: return {'message': 'invalid private key', 'code': 104031} variables = extra.get('extraVars') or {} # variables.update({'node': inventory_record.get('limit')}) if type(variables) in [dict, list]: variables = yaml.safe_dump(variables) app = template.get('app') if app: # @todo status=1 app_record = Application.find_by_id(app) if not app_record: return {'message': 'invalid app', 'code': 104043} app_type = app_record.get('type') app_params = app_record.get('params') integration = Integration(app_type, app_params) check_app = integration.check_app_params() if not check_app: return {'message': 'invalid app', 'code': 104014} job_space = integration.get_job_space(name) if job_space and variables: tpl = Template(variables) variables = tpl.render(ECLOGUE_JOB_SPACE=job_space) if variables: variables = yaml.safe_load(variables) variables and extra_vars.update(variables) options = dict() extra_options = template.get('extraOptions') if extra_options and type(extra_options) == dict: options = extra_options.copy() # default_options = get_default_options('playbook') # for key, value in extra_options.items(): # if default_options.get(key): # options[key] = value options['skip_tags'] = template.get('skip_tags', []) options['inventory'] = inventory_record # @todo limit # options['limit'] = inventory_record.get('limit', None) # options['credential'] = template.get('credential') options['limit'] = template.get('limit', 'all') options['forks'] = template.get('forks', 5) options['tags'] = template.get('tags', []) options['listtags'] = template.get('listtags', False) options['listtasks'] = template.get('listtasks', False) options['timeout'] = template.get('timeout', 10) options['verbosity'] = template.get('verbosity', 0) become_method = template.get('become_method') if become_method: options['become'] = 'yes' options['become_method'] = become_method if template.get('become_user'): options['become_user'] = template.get('become_user') vault_pass = template.get('vault_pass') if vault_pass: vault_record = Credential.find_one({'_id': ObjectId(vault_pass)}) if not vault_record or not vault_record.get('body'): return {'message': 'invalid vault pass', 'code': 104004} vault_body = vault_record.get('body') vault = Vault({'vault_pass': config.vault.get('secret')}) options['vault_pass'] = vault.decrypt_string( vault_body.get('vault_pass')) options['verbosity'] = template.get('verbosity', 0) options['diff'] = template.get('diff', False) # options['vault'] = template.get('vault') options['extra_vars'] = extra_vars status = int(extra.get('status', 0)) return { 'message': 'ok', 'data': { 'inventory': options['inventory'], 'options': options, 'name': name, 'entry': entry_record['name'], 'book_id': book_id, 'book_name': book_record['name'], 'roles': role_names, 'inventory_type': inventory_type, 'private_key': private_key, 'template': template, 'extra': extra, 'status': status, } }
def test_upload(self): data = self.get_data('book') playbook = self.get_data('playbook') data['name'] = str(uuid.uuid4()) result = Book.insert_one(data.copy()) book_id = result.inserted_id playbook['book_id'] = str(book_id) Playbook.insert_one(playbook) self.trash += [ [Book, book_id], [Playbook, playbook['_id']], ] headers = self.jwt_headers.copy() headers.update({'Content-Type': 'multipart/form-data'}) url = self.get_api_path('/playbook/upload') response = self.client.post(url, headers=headers) self.assert400(response) self.assertResponseCode(response, 104000) url = self.get_api_path('/playbook/upload') form = {'parent': '/', 'bookId': str(ObjectId())} response = self.client.post(url, data=form, headers=headers) self.assert400(response) self.assertResponseCode(response, 104001) stream = BytesIO(bytes('mock test', 'utf-8')) form['files'] = (stream, 'test.yaml') response = self.client.post(url, data=form, headers=headers) self.assert404(response) self.assertResponseCode(response, 104040) params = form.copy() stream = BytesIO(bytes('mock test', 'utf-8')) params['files'] = (stream, 'test.yaml') params['parent'] = ObjectId() response = self.client.post(url, data=params, headers=headers, content_type='multipart/form-data') self.assert400(response) self.assertResponseCode(response, 104004) stream = BytesIO(bytes('mock test', 'utf-8')) form['files'] = (stream, 'test.yaml') form['bookId'] = book_id response = self.client.post(url, data=form, headers=headers, content_type='multipart/form-data') self.assert200(response) record = Playbook.find_one({ 'book_id': str(book_id), '_id': { '$ne': playbook['_id'] } }) assert record is not None db.fs().delete(record.get('file_id')) with patch('eclogue.api.playbook.is_edit') as build_mock: build_mock.return_value = False stream = BytesIO(bytes('mock test', 'utf-8')) form['files'] = (stream, 'binary.mock') response = self.client.post(url, data=form, headers=headers, content_type='multipart/form-data') self.assert200(response) record = Playbook.find_one({ 'book_id': str(book_id), 'file_id': { '$exists': True } }) assert record is not None db.fs().delete(record.get('file_id')) Playbook().collection.delete_many({'book_id': str(book_id)})