def get_variables(self, ids, decode=False): records = self.find_by_ids(ids) variables = {} vault = Vault({'vault_pass': config.vault.get('secret')}) for record in records: config_vars = record.get('variables') if not config_vars: continue for k, v in config_vars.items(): key = '_'.join( ['ECLOGUE', 'CONFIG', record.get('name', ''), k]) if not decode: variables[key] = v continue is_encrypt = Vault.is_encrypted(v) value = v if is_encrypt: value = vault.decrypt_string(value) variables[key] = value return variables
def load_from_dir(home_path, exclude=['*.retry'], links=False, book_name=None): bucket = [] cursor = 0 parent = home_path book_name = book_name or os.path.basename(home_path) pattern = '|'.join(exclude).replace('*', '.*?') for current, dirs, files in os.walk(home_path, topdown=True, followlinks=links): pathname = current.replace(home_path, '') or '/' if exclude: match = re.search(pattern, pathname) if match: continue dir_record = { 'book_name': book_name, 'path': pathname, 'is_dir': True, 'is_edit': False, 'seq_no': cursor, 'parent': None, 'created_at': int(time.time()), } if not current == home_path: dir_record['parent'] = parent meta = get_meta(pathname=pathname) dir_record.update(meta) parent = pathname bucket.append(dir_record) for file in files: pathname = parent.rstrip('/') + '/' + file if exclude: match = re.match(pattern, pathname) if match: continue cursor += 1 filename = current + '/' + file is_edit = Dumper.is_read(filename) file_record = dir_record.copy() file_record['is_edit'] = is_edit file_record['path'] = pathname file_record['parent'] = parent file_record['is_dir'] = False file_record['seq_no'] = cursor if is_edit: with open(filename, 'r', encoding='utf-8') as fd: file_record['content'] = fd.read() file_record['md5'] = md5(file_record['content']) file_record['is_encrypt'] = Vault.is_encrypted( file_record['content']) meta = get_meta(file_record['path']) file_record.update(meta) file_record['meta'] = meta bucket.append(file_record) cursor += 1 return bucket
def parse_register(record): register = record.get('register') if not register: return record c_ids = map(lambda i: ObjectId(i), register) cfg_records = Configuration.find({'_id': {'$in': list(c_ids)}}) if not cfg_records: return record try: variables = {} content = yaml.safe_load(record.get('content', '')) if not content: return record vault = Vault({'vault_pass': config.vault.get('secret')}) for cfg in cfg_records: config_vars = cfg.get('variables') if not config_vars: continue for k, v in config_vars.items(): key = '_'.join( ['ECLOGUE', 'CONFIG', cfg.get('name', ''), k]) is_encrypt = Vault.is_encrypted(v) value = v if is_encrypt: value = vault.decrypt_string(value) variables[key] = value content = dict(content) content.update(variables) record['content'] = yaml.safe_dump(content) except Exception as e: print(e) return record
def upload_playbook(_id): files = request.files record = Book.find_by_id(_id) if not record: return jsonify({ "message": "book not found", "code": 104004, }), 400 if not files: return jsonify({ 'message': 'invalid files params', 'code': 104001 }), 400 file = files['file'] filename = file.filename.lstrip('/') path_list = filename.split('/') filename = '/'.join(path_list[1:]) filename = '/' + filename home_path, basename = os.path.split(filename) file_list = set(_make_path(filename)) for dirname in file_list: check = Playbook.find_one({ 'book_id': _id, 'path': dirname, }) if not check: parent_path, name = os.path.split(dirname) parent_path = parent_path if parent_path != '/' else None parent = { 'path': dirname, 'is_dir': True, 'is_edit': False, 'book_id': _id, 'parent': parent_path, 'name': name, 'created_at': time.time(), } meta = get_meta(dirname) parent.update(meta) parent['additions'] = meta Playbook.insert_one(parent) data = { 'path': filename, 'is_dir': False, 'parent': home_path or None, 'book_id': _id } can_edit = is_edit(file) if not can_edit: file_id = db.save_file(filename=filename, fileobj=file) data['file_id'] = file_id else: content = file.stream.read() content = content.decode('utf-8') data['is_encrypt'] = Vault.is_encrypted(content) if data['is_encrypt']: # @todo vault password vault = Vault() data['content'] = vault.encrypt_string(content) data['md5'] = md5(content) else: data['content'] = content data['md5'] = md5(content) meta = get_meta(data['path']) data.update(meta) data['additions'] = meta data['is_edit'] = can_edit data['created_at'] = time.time() data['updated_at'] = time.time() Playbook.update_one({ 'path': filename, 'book_id': _id }, {'$set': data}, upsert=True) return jsonify({ "message": "ok", "code": 0, })
def import_book_from_dir(self, home_path, book_id, exclude=None, links=False, prefix='/'): """ import dir file to db @todo """ book_id = str(book_id) exclude = exclude or ['*.retry'] bucket = [] cursor = 0 home_path = home_path.rstrip('/') parent = home_path book_record = Book.find_by_id(book_id) model = Model.build_model('playbook') playbooks = model.find({'book_id': book_id}) paths = map(lambda i: i['path'], playbooks) paths = list(paths) pattern = '|'.join(exclude).replace('*', '.*?') home_path = '/'.join([home_path, '']) for current, dirs, files in os.walk(home_path, topdown=True, followlinks=links): pathname = current.replace(home_path, '') if pathname != '/': pathname = os.path.join(prefix, pathname) if exclude: match = re.search(pattern, pathname) if match: continue if pathname in paths: index = paths.index(pathname) paths.pop(index) dir_record = { 'book_id': str(book_record.get('_id')), 'path': pathname, 'is_dir': True, 'is_edit': False, 'seq_no': cursor, 'parent': None, 'created_at': int(time.time()), } if not current == home_path: dir_record['parent'] = parent meta = get_meta(pathname) dir_record.update(meta) dir_record['additions'] = meta parent = pathname bucket.append(dir_record) for file in files: pathname = parent.rstrip('/') + '/' + file if exclude: match = re.match(pattern, pathname) if match: continue cursor += 1 filename = current + '/' + file can_edit = is_edit(filename) file_record = dir_record.copy() file_record['is_edit'] = can_edit file_record['path'] = pathname file_record['parent'] = parent file_record['is_dir'] = False file_record['seq_no'] = cursor if is_edit: with open(filename, 'r', encoding='utf-8') as fd: file_record['content'] = fd.read() file_record['md5'] = md5(file_record['content']) file_record['is_encrypt'] = Vault.is_encrypted( file_record['content']) meta = get_meta(file_record['path']) file_record['additions'] = meta file_record.update(meta) bucket.append(file_record) cursor += 1 is_entry = filter(lambda i: i.get('role') == 'entry', bucket) is_entry = list(is_entry) # if not entry set book status to disable if not is_entry: Book.update_one({'_id': ObjectId(book_id)}, {'$set': { 'status': 0 }}) for path in paths: model.delete_one({'book_id': book_id, 'path': path}) mapping = {} map(lambda i: {mapping['path']: i}, playbooks) for item in bucket: record = mapping.get(item['path']) if not record: model.insert_one(item) continue else: # inherit old additions if record['additions']: item['additions'].update(record['additions']) model.update_one({'_id': record['_id']}, {'$set': item}) return bucket
def update_credential(_id): payload = request.get_json() current_user = login_user allow_types = [ 'vault_pass', 'private_key', 'token', ] record = db.collection('credentials').find_one({'_id': ObjectId(_id)}) if not record: return jsonify({ 'message': 'record not found', 'code': 134040, }), 404 data = {} description = payload.get('description') name = payload.get('name') status = payload.get('status') if name and record.get('name') != name: data['name'] = name existed = db.collection('credentials').find_one({'name': name}) if existed: return jsonify({ 'message': 'name already existed', 'code': 134002 }), 400 if status is not None: data['status'] = int(status) if description: data['description'] = description credential_type = payload.get('type') if credential_type: data['type'] = credential_type if credential_type not in allow_types: return jsonify({ 'message': 'invalid type', 'code': 134001 }), 400 body = payload.get('body') if not body or not body.get(credential_type): return jsonify({ 'message': 'illegal credential params', 'code': 134002 }), 400 is_encrypt = Vault.is_encrypted(body[credential_type]) if not is_encrypt: body[credential_type] = encrypt_credential(body[credential_type]) data['body'] = body scope = payload.get('scope', 'global') data['scope'] = scope users = payload.get('users', [login_user.get('username')]) user_list = db.collection('users').find({'username': {'$in': users}}) user_list = list(user_list) data['maintainer'] = [] for item in user_list: data['maintainer'].append(item.get('username')) Credential.update_one({'_id': record['_id']}, {'$set': data}) return jsonify({ 'message': 'ok', 'code': 0, })
def import_book_from_dir(self, home_path, book_id, exclude=['*.retry'], links=False): bucket = [] cursor = 0 parent = home_path book_record = Book.find_one({'_id': ObjectId(book_id)}) pattern = '|'.join(exclude).replace('*', '.*?') for current, dirs, files in os.walk(home_path, topdown=True, followlinks=links): pathname = current.replace(home_path, '') or '/' if exclude: match = re.search(pattern, pathname) if match: continue dir_record = { 'book_id': str(book_record.get('_id')), 'path': pathname, 'is_dir': True, 'is_edit': False, 'seq_no': cursor, 'parent': None, 'created_at': int(time.time()), } if not current == home_path: dir_record['parent'] = parent meta = Workspace.get_meta(pathname=pathname) dir_record.update(meta) dir_record['additions'] = meta parent = pathname bucket.append(dir_record) for file in files: pathname = parent.rstrip('/') + '/' + file if exclude: match = re.match(pattern, pathname) if match: continue cursor += 1 filename = current + '/' + file can_edit = is_edit(filename) file_record = dir_record.copy() file_record['is_edit'] = can_edit file_record['path'] = pathname file_record['parent'] = parent file_record['is_dir'] = False file_record['seq_no'] = cursor if is_edit: with open(filename, 'r', encoding='utf-8') as fd: file_record['content'] = fd.read() file_record['md5'] = md5(file_record['content']) file_record['is_encrypt'] = Vault.is_encrypted( file_record['content']) meta = self._get_role(file_record['path']) file_record.update(meta) file_record['additions'] = meta bucket.append(file_record) cursor += 1 is_entry = filter(lambda i: i.get('role') == 'entry', bucket) is_entry = list(is_entry) if not is_entry: path = '/entry.yml' entry = { 'book_id': str(book_record.get('_id')), 'path': path, 'is_dir': False, 'is_edit': True, 'seq_no': 0, 'content': '', 'parent': None, 'created_at': int(time.time()), } meta = self._get_role(path) entry.update(meta) entry['additions'] = meta bucket.append(entry) return bucket