def get_variables(self, ids, decode=False):
records = self.find_by_ids(ids)
variables = {}
vault = Vault({'vault_pass': config.vault.get('secret')})
for record in records:
config_vars = record.get('variables')
if not config_vars:
continue
for k, v in config_vars.items():
key = '_'.join(
['ECLOGUE', 'CONFIG',
record.get('name', ''), k])
if not decode:
variables[key] = v
continue
is_encrypt = Vault.is_encrypted(v)
value = v
if is_encrypt:
value = vault.decrypt_string(value)
variables[key] = value
return variables
def load_from_dir(home_path,
exclude=['*.retry'],
links=False,
book_name=None):
bucket = []
cursor = 0
parent = home_path
book_name = book_name or os.path.basename(home_path)
pattern = '|'.join(exclude).replace('*', '.*?')
for current, dirs, files in os.walk(home_path,
topdown=True,
followlinks=links):
pathname = current.replace(home_path, '') or '/'
if exclude:
match = re.search(pattern, pathname)
if match:
continue
dir_record = {
'book_name': book_name,
'path': pathname,
'is_dir': True,
'is_edit': False,
'seq_no': cursor,
'parent': None,
'created_at': int(time.time()),
}
if not current == home_path:
dir_record['parent'] = parent
meta = get_meta(pathname=pathname)
dir_record.update(meta)
parent = pathname
bucket.append(dir_record)
for file in files:
pathname = parent.rstrip('/') + '/' + file
if exclude:
match = re.match(pattern, pathname)
if match:
continue
cursor += 1
filename = current + '/' + file
is_edit = Dumper.is_read(filename)
file_record = dir_record.copy()
file_record['is_edit'] = is_edit
file_record['path'] = pathname
file_record['parent'] = parent
file_record['is_dir'] = False
file_record['seq_no'] = cursor
if is_edit:
with open(filename, 'r', encoding='utf-8') as fd:
file_record['content'] = fd.read()
file_record['md5'] = md5(file_record['content'])
file_record['is_encrypt'] = Vault.is_encrypted(
file_record['content'])
meta = get_meta(file_record['path'])
file_record.update(meta)
file_record['meta'] = meta
bucket.append(file_record)
cursor += 1
return bucket
def parse_register(record):
register = record.get('register')
if not register:
return record
c_ids = map(lambda i: ObjectId(i), register)
cfg_records = Configuration.find({'_id': {'$in': list(c_ids)}})
if not cfg_records:
return record
try:
variables = {}
content = yaml.safe_load(record.get('content', ''))
if not content:
return record
vault = Vault({'vault_pass': config.vault.get('secret')})
for cfg in cfg_records:
config_vars = cfg.get('variables')
if not config_vars:
continue
for k, v in config_vars.items():
key = '_'.join(
['ECLOGUE', 'CONFIG',
cfg.get('name', ''), k])
is_encrypt = Vault.is_encrypted(v)
value = v
if is_encrypt:
value = vault.decrypt_string(value)
variables[key] = value
content = dict(content)
content.update(variables)
record['content'] = yaml.safe_dump(content)
except Exception as e:
print(e)
return record
def upload_playbook(_id):
files = request.files
record = Book.find_by_id(_id)
if not record:
return jsonify({
"message": "book not found",
"code": 104004,
}), 400
if not files:
return jsonify({
'message': 'invalid files params',
'code': 104001
}), 400
file = files['file']
filename = file.filename.lstrip('/')
path_list = filename.split('/')
filename = '/'.join(path_list[1:])
filename = '/' + filename
home_path, basename = os.path.split(filename)
file_list = set(_make_path(filename))
for dirname in file_list:
check = Playbook.find_one({
'book_id': _id,
'path': dirname,
})
if not check:
parent_path, name = os.path.split(dirname)
parent_path = parent_path if parent_path != '/' else None
parent = {
'path': dirname,
'is_dir': True,
'is_edit': False,
'book_id': _id,
'parent': parent_path,
'name': name,
'created_at': time.time(),
}
meta = get_meta(dirname)
parent.update(meta)
parent['additions'] = meta
Playbook.insert_one(parent)
data = {
'path': filename,
'is_dir': False,
'parent': home_path or None,
'book_id': _id
}
can_edit = is_edit(file)
if not can_edit:
file_id = db.save_file(filename=filename, fileobj=file)
data['file_id'] = file_id
else:
content = file.stream.read()
content = content.decode('utf-8')
data['is_encrypt'] = Vault.is_encrypted(content)
if data['is_encrypt']:
# @todo vault password
vault = Vault()
data['content'] = vault.encrypt_string(content)
data['md5'] = md5(content)
else:
data['content'] = content
data['md5'] = md5(content)
meta = get_meta(data['path'])
data.update(meta)
data['additions'] = meta
data['is_edit'] = can_edit
data['created_at'] = time.time()
data['updated_at'] = time.time()
Playbook.update_one({
'path': filename,
'book_id': _id
}, {'$set': data},
upsert=True)
return jsonify({
"message": "ok",
"code": 0,
})
def import_book_from_dir(self,
home_path,
book_id,
exclude=None,
links=False,
prefix='/'):
"""
import dir file to db
@todo
"""
book_id = str(book_id)
exclude = exclude or ['*.retry']
bucket = []
cursor = 0
home_path = home_path.rstrip('/')
parent = home_path
book_record = Book.find_by_id(book_id)
model = Model.build_model('playbook')
playbooks = model.find({'book_id': book_id})
paths = map(lambda i: i['path'], playbooks)
paths = list(paths)
pattern = '|'.join(exclude).replace('*', '.*?')
home_path = '/'.join([home_path, ''])
for current, dirs, files in os.walk(home_path,
topdown=True,
followlinks=links):
pathname = current.replace(home_path, '')
if pathname != '/':
pathname = os.path.join(prefix, pathname)
if exclude:
match = re.search(pattern, pathname)
if match:
continue
if pathname in paths:
index = paths.index(pathname)
paths.pop(index)
dir_record = {
'book_id': str(book_record.get('_id')),
'path': pathname,
'is_dir': True,
'is_edit': False,
'seq_no': cursor,
'parent': None,
'created_at': int(time.time()),
}
if not current == home_path:
dir_record['parent'] = parent
meta = get_meta(pathname)
dir_record.update(meta)
dir_record['additions'] = meta
parent = pathname
bucket.append(dir_record)
for file in files:
pathname = parent.rstrip('/') + '/' + file
if exclude:
match = re.match(pattern, pathname)
if match:
continue
cursor += 1
filename = current + '/' + file
can_edit = is_edit(filename)
file_record = dir_record.copy()
file_record['is_edit'] = can_edit
file_record['path'] = pathname
file_record['parent'] = parent
file_record['is_dir'] = False
file_record['seq_no'] = cursor
if is_edit:
with open(filename, 'r', encoding='utf-8') as fd:
file_record['content'] = fd.read()
file_record['md5'] = md5(file_record['content'])
file_record['is_encrypt'] = Vault.is_encrypted(
file_record['content'])
meta = get_meta(file_record['path'])
file_record['additions'] = meta
file_record.update(meta)
bucket.append(file_record)
cursor += 1
is_entry = filter(lambda i: i.get('role') == 'entry', bucket)
is_entry = list(is_entry)
# if not entry set book status to disable
if not is_entry:
Book.update_one({'_id': ObjectId(book_id)},
{'$set': {
'status': 0
}})
for path in paths:
model.delete_one({'book_id': book_id, 'path': path})
mapping = {}
map(lambda i: {mapping['path']: i}, playbooks)
for item in bucket:
record = mapping.get(item['path'])
if not record:
model.insert_one(item)
continue
else:
# inherit old additions
if record['additions']:
item['additions'].update(record['additions'])
model.update_one({'_id': record['_id']}, {'$set': item})
return bucket
def update_credential(_id):
payload = request.get_json()
current_user = login_user
allow_types = [
'vault_pass',
'private_key',
'token',
]
record = db.collection('credentials').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 134040,
}), 404
data = {}
description = payload.get('description')
name = payload.get('name')
status = payload.get('status')
if name and record.get('name') != name:
data['name'] = name
existed = db.collection('credentials').find_one({'name': name})
if existed:
return jsonify({
'message': 'name already existed',
'code': 134002
}), 400
if status is not None:
data['status'] = int(status)
if description:
data['description'] = description
credential_type = payload.get('type')
if credential_type:
data['type'] = credential_type
if credential_type not in allow_types:
return jsonify({
'message': 'invalid type',
'code': 134001
}), 400
body = payload.get('body')
if not body or not body.get(credential_type):
return jsonify({
'message': 'illegal credential params',
'code': 134002
}), 400
is_encrypt = Vault.is_encrypted(body[credential_type])
if not is_encrypt:
body[credential_type] = encrypt_credential(body[credential_type])
data['body'] = body
scope = payload.get('scope', 'global')
data['scope'] = scope
users = payload.get('users', [login_user.get('username')])
user_list = db.collection('users').find({'username': {'$in': users}})
user_list = list(user_list)
data['maintainer'] = []
for item in user_list:
data['maintainer'].append(item.get('username'))
Credential.update_one({'_id': record['_id']}, {'$set': data})
return jsonify({
'message': 'ok',
'code': 0,
})
def import_book_from_dir(self,
home_path,
book_id,
exclude=['*.retry'],
links=False):
bucket = []
cursor = 0
parent = home_path
book_record = Book.find_one({'_id': ObjectId(book_id)})
pattern = '|'.join(exclude).replace('*', '.*?')
for current, dirs, files in os.walk(home_path,
topdown=True,
followlinks=links):
pathname = current.replace(home_path, '') or '/'
if exclude:
match = re.search(pattern, pathname)
if match:
continue
dir_record = {
'book_id': str(book_record.get('_id')),
'path': pathname,
'is_dir': True,
'is_edit': False,
'seq_no': cursor,
'parent': None,
'created_at': int(time.time()),
}
if not current == home_path:
dir_record['parent'] = parent
meta = Workspace.get_meta(pathname=pathname)
dir_record.update(meta)
dir_record['additions'] = meta
parent = pathname
bucket.append(dir_record)
for file in files:
pathname = parent.rstrip('/') + '/' + file
if exclude:
match = re.match(pattern, pathname)
if match:
continue
cursor += 1
filename = current + '/' + file
can_edit = is_edit(filename)
file_record = dir_record.copy()
file_record['is_edit'] = can_edit
file_record['path'] = pathname
file_record['parent'] = parent
file_record['is_dir'] = False
file_record['seq_no'] = cursor
if is_edit:
with open(filename, 'r', encoding='utf-8') as fd:
file_record['content'] = fd.read()
file_record['md5'] = md5(file_record['content'])
file_record['is_encrypt'] = Vault.is_encrypted(
file_record['content'])
meta = self._get_role(file_record['path'])
file_record.update(meta)
file_record['additions'] = meta
bucket.append(file_record)
cursor += 1
is_entry = filter(lambda i: i.get('role') == 'entry', bucket)
is_entry = list(is_entry)
if not is_entry:
path = '/entry.yml'
entry = {
'book_id': str(book_record.get('_id')),
'path': path,
'is_dir': False,
'is_edit': True,
'seq_no': 0,
'content': '',
'parent': None,
'created_at': int(time.time()),
}
meta = self._get_role(path)
entry.update(meta)
entry['additions'] = meta
bucket.append(entry)
return bucket