def test_delete_book(self):
data = self.get_data('book')
playbook = self.get_data('playbook')
data['name'] = str(uuid.uuid4())
result = Book.insert_one(data.copy())
book_id = result.inserted_id
playbook['book_id'] = str(book_id)
Playbook.insert_one(playbook)
self.trash += [
[Book, book_id],
[Playbook, playbook['_id']],
]
url = self.get_api_path('/books/' + str(book_id))
not_found_url = self.get_api_path('/books/' + str(ObjectId()))
response = self.client.delete(not_found_url, headers=self.jwt_headers)
self.assert404(response)
self.assertResponseCode(response, 154041)
response = self.client.delete(url, headers=self.jwt_headers)
self.assert200(response)
record = Book.find_by_id(book_id)
self.assertEqual(record, None)
record = Book().collection.find_one({'_id': book_id})
self.assertIsNotNone(record)
self.assertEqual(record['_id'], book_id)
self.assertEqual(record['status'], -1)
playbook_record = Playbook().collection.find_one(
{'_id': playbook['_id']})
self.assertIsNotNone(playbook_record)
self.assertEqual(playbook_record['status'], -1)
def book_detail(_id):
record = Book.find_by_id(_id)
if not record:
return jsonify({'message': 'record not found', 'code': 154041}), 400
return jsonify({
'message': 'ok',
'code': 0,
'data': record,
})
def dispatch(book_id, entry, payload):
print('xxxx', book_id, entry, payload)
book = Book.find_by_id(book_id)
if not book:
return False
username = payload.get('username')
run_id = payload.get('req_id') or str(uuid.uuid4())
params = [book_id, run_id]
options = payload.get('options')
if not entry:
return False
if type(options) == str:
args = options.split(' ')
pb = PlaybookCLI(args)
pb.init_parser()
options, args = pb.parser.parse_args(args[1:])
options, args = pb.post_process_args(options, args)
options = options.__dict__
options['entry'] = args
for i in options['inventory']:
if not os.path.isfile(i):
i = os.path.basename(i)
options['inventory'] = i
break
queue_name = 'book_runtime'
func = run
task = Task(tiger,
func=func,
args=params,
kwargs=options,
queue=queue_name,
unique=True,
lock=True,
lock_key=book_id)
run_record = {
'book_id': book_id,
'run_id': run_id,
'run_by': username,
'options': options,
'result': '',
'state': 'pending',
'created_at': 1,
'updated_at': 2,
}
result = Perform.insert_one(run_record)
task.delay()
return result.inserted_id
def download_book(_id):
record = Book.find_by_id(_id)
if not record:
return jsonify({'message': 'record not found', 'code': 104040}), 404
name = record.get('name')
wk = Workspace()
with build_book_from_db(name) as bookspace:
filename = name + '.zip'
with NamedTemporaryFile('w+t', delete=False) as fd:
make_zip(bookspace, fd.name)
return send_file(fd.name,
attachment_filename=filename,
as_attachment=True)
def get_playbook(_id):
book = Book.find_by_id(_id)
if not book or int(book.get('status') == -1):
return jsonify({
'message': 'invalid id',
'code': 154001,
}), 400
cursor = Playbook.find({'book_id': str(book.get('_id'))})
cursor = cursor.sort([('is_edit', pymongo.ASCENDING),
('path', pymongo.ASCENDING)])
# for item in cursor:
# db.collection('playbook').update_one({'_id': item['_id']}, {'$set': {'book_id': str(item.get('book_id'))}})
return jsonify({
'message': 'ok',
'code': 0,
'data': list(cursor),
})
def run(_id):
book = Book.find_by_id(_id)
if not book:
return jsonify({'message': 'record not found', 'code': 10404}), 404
payload = request.get_json()
options = payload.get('options')
entry = options.get('entry')
args = options.get('args')
req_id = str(current_request_id())
params = {
'username': login_user.get('username'),
'req_id': req_id,
'args': args,
'options': options
}
result = dispatch(_id, entry, params)
if not result:
return jsonify({'message': 'invalid request', 'code': 104008}), 400
return jsonify({'message': 'ok', 'code': 0, 'data': {'taskId': result}})
def delete_book(_id):
record = Book.find_by_id(_id)
if not record:
return jsonify({'message': 'record not found', 'code': 154041}), 404
update = {
'$set': {
'status': -1,
'delete_at': time.time(),
'delete_by': login_user.get('username'),
'version': str(ObjectId()),
}
}
Book.update_one({'_id': record['_id']}, update=update)
db.collection('playbook').update_many({'book_id': str(record['_id'])},
update=update)
return jsonify({
'message': 'ok',
'code': 0,
})
def lint_book(_id):
# lint book use ansibleint
book = Book.find_by_id(_id)
if not book:
return jsonify({'message': 'record not found', 'code': 104040}), 404
body = request.get_json() or {}
tags = body.get('tags')
skiptags = body.get('skiptags')
tasks = body.get('tasks')
roles = body.get('roles')
opts = dict()
if tags:
opts['tags'] = tags
if tasks:
opts['tasks'] = tasks
if roles:
opts['roles'] = roles
if skiptags:
opts['skiptags'] = skiptags
result = lint(_id, options=opts)
return jsonify({'message': 'ok', 'code': 0, 'data': result})
def upload_playbook(_id):
files = request.files
record = Book.find_by_id(_id)
if not record:
return jsonify({
"message": "book not found",
"code": 104004,
}), 400
if not files:
return jsonify({
'message': 'invalid files params',
'code': 104001
}), 400
file = files['file']
filename = file.filename.lstrip('/')
path_list = filename.split('/')
filename = '/'.join(path_list[1:])
filename = '/' + filename
home_path, basename = os.path.split(filename)
file_list = set(_make_path(filename))
for dirname in file_list:
check = Playbook.find_one({
'book_id': _id,
'path': dirname,
})
if not check:
parent_path, name = os.path.split(dirname)
parent_path = parent_path if parent_path != '/' else None
parent = {
'path': dirname,
'is_dir': True,
'is_edit': False,
'book_id': _id,
'parent': parent_path,
'name': name,
'created_at': time.time(),
}
meta = get_meta(dirname)
parent.update(meta)
parent['additions'] = meta
Playbook.insert_one(parent)
data = {
'path': filename,
'is_dir': False,
'parent': home_path or None,
'book_id': _id
}
can_edit = is_edit(file)
if not can_edit:
file_id = db.save_file(filename=filename, fileobj=file)
data['file_id'] = file_id
else:
content = file.stream.read()
content = content.decode('utf-8')
data['is_encrypt'] = Vault.is_encrypted(content)
if data['is_encrypt']:
# @todo vault password
vault = Vault()
data['content'] = vault.encrypt_string(content)
data['md5'] = md5(content)
else:
data['content'] = content
data['md5'] = md5(content)
meta = get_meta(data['path'])
data.update(meta)
data['additions'] = meta
data['is_edit'] = can_edit
data['created_at'] = time.time()
data['updated_at'] = time.time()
Playbook.update_one({
'path': filename,
'book_id': _id
}, {'$set': data},
upsert=True)
return jsonify({
"message": "ok",
"code": 0,
})
def get_job(_id):
username = login_user.get('username')
is_admin = login_user.get('is_admin')
job = Job.find_by_id(_id)
if not job:
return jsonify({
'message': 'record not found',
'code': 154001,
}), 400
if not is_admin and username not in job['maintainer']:
return jsonify({'message': 'permission deny', 'code': 104031}), 403
template = job.get('template')
inventory_type = template.get('inventory_type')
inventory = template.get('inventory')
if job.get('type') == 'adhoc':
inventory_content = parse_cmdb_inventory(inventory)
logs = None
task = Task.find_one({'job_id': _id})
if task:
log = db.collection('logs').find_one({'task_id': str(task['_id'])})
if log:
logs = log.get('message')
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
'logs': logs
},
})
if inventory_type == 'file':
inventory_content = parse_file_inventory(inventory)
else:
inventory_content = parse_cmdb_inventory(inventory)
check_playbook(job['book_id'])
if inventory_type == 'file':
book = Book.find_by_id(job['book_id'])
if not book:
hosts = []
else:
hosts = get_inventory_by_book(book.get('_id'),
book_name=book.get('name'))
else:
hosts = get_inventory_from_cmdb()
roles = []
condition = {
'book_id': str(job['book_id']),
'role': 'roles',
'is_dir': True
}
parent = Playbook.find_one(condition)
if parent:
where = {
'book_id': job['book_id'],
'is_dir': True,
'parent': parent.get('path')
}
cursor = Playbook.find(where)
roles = list(cursor)
logs = None
task = Task.find_one({'job_id': _id})
if task:
log = db.collection('logs').find_one({'task_id': str(task['_id'])})
if log:
logs = log.get('message')
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
'hosts': hosts,
'roles': roles,
'logs': logs,
},
})
def run(book_id, run_id, **options):
perform = Perform.find_one({'run_id': run_id})
book = Book.find_by_id(book_id)
start_at = time.time()
state = 'progressing'
result = ''
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stderr = sys.stdout = temp_stdout = Reporter(run_id)
try:
if book.get('repo') == 'git':
vcs = GitDownload(book.get('repo_options'))
install_path = vcs.install()
# @todo
book_name = book.get('name')
with build_book_from_db(book_name, build_id=run_id) as dest:
if not dest:
result = 'install book failed'
logger.warning(result)
state = 'finish'
else:
inventory = options['inventory']
if type(inventory == str):
inventory = os.path.join(dest, inventory)
entry = os.path.join(dest, options['entry'].pop())
if type(options['tags']) == str:
options['tags'] = options['tags'].split(',')
options['basedir'] = dest
print('xxxxxxxxxxxxx????', inventory, options)
runner = PlayBookRunner(inventory, options)
runner.run([entry])
result = runner.get_result()
print('result', result)
state = 'finish'
except Exception as err:
result = str(err)
extra = {'run_id': run_id}
logger.error('run task with exception: {}'.format(result), extra=extra)
state = 'error'
raise err
finally:
content = temp_stdout.getvalue()
temp_stdout.close(True)
sys.stdout = old_stdout
sys.stderr = old_stderr
print(content)
finish_at = time.time()
update = {
'$set': {
'start_at': start_at,
'finish_at': finish_at,
'state': state,
'duration': finish_at - start_at,
'result': str(result),
'trace': content,
}
}
Perform.update_one({'_id': perform['_id']}, update=update)
def add_book():
params = request.get_json() or request.form
if not params:
return jsonify({
'message': 'invalid params',
'code': 154000,
}), 400
name = params.get('name')
if not name:
return jsonify({
'message': 'name param required',
'code': 154001,
}), 400
existed = Book.find_one({'name': name})
if existed:
return jsonify({
'message': 'book exist',
'code': 154003,
}), 400
description = params.get('description')
status = params.get('status', 0)
book_id = params.get('_id')
import_type = params.get('importType')
repo = params.get('repo')
maintainer = params.get('maintainer', [])
readonly = params.get('readonly', False)
if book_id:
record = Book.find_by_id(book_id)
if not record:
return jsonify({
'message': 'record not found',
'code': 154041,
}), 404
else:
data = {
'name': name,
'readonly': readonly,
'description': description,
'maintainer': maintainer,
'import_type': import_type,
'repo': repo,
'created_at': int(time.time())
}
result = Book.insert_one(data)
book_id = result.inserted_id
if import_type == 'galaxy' and repo:
galaxy = AnsibleGalaxy([repo])
galaxy.install(book_id)
logger.info('import galaxy', extra={'repo': repo})
elif import_type == 'git' and repo:
git = GitDownload({'repository': repo})
dest = git.install()
Workspace().import_book_from_dir(dest, book_id, exclude=['.git'])
data = {
'readonly': readonly,
'description': description,
'maintainer': maintainer,
'import_type': import_type,
'repo': repo,
'status': int(status),
'updated_at': time.time(),
}
Book.update_one({'_id': ObjectId(book_id)}, {'$set': data}, upsert=True)
data['_id'] = book_id
return jsonify({
'message': 'ok',
'code': 0,
'data': data,
})
def playbook(payload):
payload = payload or {}
print(payload)
book_id = payload.get('book_id')
cmd = payload.get('cmd')
if not book_id and not cmd:
emit('playbook', {'code': 1404, 'message': 'invalid params'})
book = Book.find_by_id(book_id)
if not book:
emit('playbook', {'code': 1404, 'message': 'book not found'})
args = cmd.lstrip().split()
allow_cmd = ['ls', 'll', 'cat', 'ansible-playbook', 'cd', 'pwd']
if args[0] not in allow_cmd:
return emit('playbook', {'code': 1403, 'message': 'invalid command'})
# with build_book_from_db(book.get('name')) as cwd:
cwd = payload.get('cwd') or ''
cwd = cwd.strip('/')
wk = Workspace()
book_space = wk.load_book_from_db(book['name'])
try:
if args[0] == 'ansible-playbook':
task_id = dispatch(
book_id, 'entry.yml',
{'options': 'ansible-playbook -i hosts entry.yml -t test'})
if not task_id:
return emit('playbook', {'message': ''})
print('fuckkkkkkkkkkkkkk', task_id)
return emit(
'book_task', {
'code': 0,
'type': 'task',
'message': 'waiting for task launch...',
'data': {
'state': 'pending',
'taskId': str(task_id),
}
})
cwd = os.path.join(book_space, cwd)
if args[0] == 'cd':
if not args[1]:
return emit({'code': 1400, 'message': 'invalid args'})
cwd = os.path.join(cwd, './' + args[1])
cwd = os.path.realpath(cwd)
if len(cwd) < len(book_space):
cwd = book_space
args = ['ls', '-a']
process = subprocess.Popen(
args,
cwd=cwd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
emit(
'playbook', {
'code': 0,
'result': {
'stdout': stdout.decode().replace(book_space, ''),
'stderr': stdout.decode().replace(book_space, ''),
'cwd': cwd.replace(book_space, '')
}
})
except Exception as err:
print(err)
emit('playbook', {
'code': 1500,
'message': str(err).replace(book_space, '')
})
def lint(book_id, options, config=None):
"""
base on ansiblelint
refer to ansiblelint.__main__.py
:param book_id:
:param options:
:param config:
:return: None
"""
formatter = formatters.Formatter()
options = get_default_options(options)
where = {
'book_id': str(book_id),
'role': 'entry',
}
entries = Playbook.find(where)
if not entries:
return False
book = Book.find_by_id(book_id)
if not book:
return False
if config:
if 'quiet' in config:
options.quiet = options.quiet or config['quiet']
if 'parseable' in config:
options.parseable = options.parseable or config['parseable']
if 'parseable_severity' in config:
options.parseable_severity = options.parseable_severity or \
config['parseable_severity']
if 'use_default_rules' in config:
options.use_default_rules = options.use_default_rules or config['use_default_rules']
if 'verbosity' in config:
options.verbosity = options.verbosity + config['verbosity']
options.exclude_paths.extend(
config.get('exclude_paths', []))
if 'rulesdir' in config:
options.rulesdir = options.rulesdir + config['rulesdir']
if 'skip_list' in config:
options.skip_list = options.skip_list + config['skip_list']
if 'tags' in config:
options.tags = options.tags + config['tags']
if options.quiet:
formatter = formatters.QuietFormatter()
if options.parseable:
formatter = formatters.ParseableFormatter()
if options.parseable_severity:
formatter = formatters.ParseableSeverityFormatter()
# no args triggers auto-detection mode
# if len(args) == 0 and not (options.listrules or options.listtags):
# args = get_playbooks_and_roles(options=options)
if options.use_default_rules:
rulesdirs = options.rulesdir + [default_rulesdir]
else:
rulesdirs = options.rulesdir or [default_rulesdir]
rules = RulesCollection()
for rulesdir in rulesdirs:
rules.extend(RulesCollection.create_from_directory(rulesdir))
if options.listrules:
return 0
if options.listtags:
return 0
if isinstance(options.tags, six.string_types):
options.tags = options.tags.split(',')
skip = set()
for s in options.skip_list:
skip.update(str(s).split(','))
options.skip_list = frozenset(skip)
with build_book_from_db(book.get('name'), options.get('roles')) as book_path:
playbooks = []
for record in entries:
entry = os.path.join(book_path, record['path'][1:])
playbooks.append(entry)
playbooks = sorted(set(playbooks))
matches = list()
checked_files = set()
for playbook in playbooks:
runner = Runner(rules, playbook, options.tags,
options.skip_list, options.exclude_paths,
options.verbosity, checked_files)
matches.extend(runner.run())
matches.sort(key=lambda x: (normpath(x.filename), x.linenumber, x.rule.id))
results = []
for match in matches:
filename = str(match.filename)
filename = filename.replace(book_path, '')
results.append({
'lineNumber': match.linenumber,
'line': str(match.line),
'rule': match.rule.id,
'filename': filename,
'message': match.message,
})
return results
def load_ansible_playbook(payload):
template = payload.get('template')
extra = payload.get('extra', {}) or {}
if not template:
return {
'message': 'invalid params',
'code': 104000,
}
name = template.get('name')
if not name:
return {
'message': 'name required',
'code': 104000,
}
entry = template.get('entry')
if len(entry) < 2:
return {
'message': 'entry not found',
'code': 104001,
}
book_id, entry_id = entry
book_record = Book.find_by_id(book_id)
if not book_record:
return {
'message': 'book not found',
'code': 1040011,
}
entry_record = Playbook.find_by_id(entry_id)
if not entry_record:
return {
'message': 'entry not found',
'code': 104001,
}
inventory_type = template.get('inventory_type', 'file')
inventory = template.get('inventory', None)
if not inventory:
return {'message': 'invalid param inventory', 'code': 104002}
if inventory_type == 'file':
inventory_record = parse_file_inventory(inventory)
else:
inventory_record = parse_cmdb_inventory(inventory)
if not inventory_record:
return {'message': 'illegal inventory', 'code': 104002}
group_name = inventory_record.keys()
if not inventory_record:
return {'message': 'invalid param inventory', 'code': 104002}
roles = template.get('roles')
role_names = []
if roles:
roles = list(map(lambda i: ObjectId(i), roles))
check = Playbook.find({
'book_id': book_id,
'_id': {
'$in': roles,
}
})
check = list(check)
if not check:
return {'message': 'invalid param role', 'code': 104003}
role_names = list(map(lambda i: i.get('name'), check))
extra_vars = {
'node': list(group_name),
}
private_key = template.get('private_key')
if private_key:
key_record = Credential.find_one({
'_id': ObjectId(private_key),
'type': 'private_key'
})
if not key_record:
return {'message': 'invalid private key', 'code': 104031}
variables = extra.get('extraVars') or {}
# variables.update({'node': inventory_record.get('limit')})
if type(variables) in [dict, list]:
variables = yaml.safe_dump(variables)
app = template.get('app')
if app:
# @todo status=1
app_record = Application.find_by_id(app)
if not app_record:
return {'message': 'invalid app', 'code': 104043}
app_type = app_record.get('type')
app_params = app_record.get('params')
integration = Integration(app_type, app_params)
check_app = integration.check_app_params()
if not check_app:
return {'message': 'invalid app', 'code': 104014}
job_space = integration.get_job_space(name)
if job_space and variables:
tpl = Template(variables)
variables = tpl.render(ECLOGUE_JOB_SPACE=job_space)
if variables:
variables = yaml.safe_load(variables)
variables and extra_vars.update(variables)
options = dict()
extra_options = template.get('extraOptions')
if extra_options and type(extra_options) == dict:
options = extra_options.copy()
# default_options = get_default_options('playbook')
# for key, value in extra_options.items():
# if default_options.get(key):
# options[key] = value
options['skip_tags'] = template.get('skip_tags', [])
options['inventory'] = inventory_record
# @todo limit
# options['limit'] = inventory_record.get('limit', None)
# options['credential'] = template.get('credential')
options['limit'] = template.get('limit', 'all')
options['forks'] = template.get('forks', 5)
options['tags'] = template.get('tags', [])
options['listtags'] = template.get('listtags', False)
options['listtasks'] = template.get('listtasks', False)
options['timeout'] = template.get('timeout', 10)
options['verbosity'] = template.get('verbosity', 0)
become_method = template.get('become_method')
if become_method:
options['become'] = 'yes'
options['become_method'] = become_method
if template.get('become_user'):
options['become_user'] = template.get('become_user')
vault_pass = template.get('vault_pass')
if vault_pass:
vault_record = Credential.find_one({'_id': ObjectId(vault_pass)})
if not vault_record or not vault_record.get('body'):
return {'message': 'invalid vault pass', 'code': 104004}
vault_body = vault_record.get('body')
vault = Vault({'vault_pass': config.vault.get('secret')})
options['vault_pass'] = vault.decrypt_string(
vault_body.get('vault_pass'))
options['verbosity'] = template.get('verbosity', 0)
options['diff'] = template.get('diff', False)
# options['vault'] = template.get('vault')
options['extra_vars'] = extra_vars
status = int(extra.get('status', 0))
return {
'message': 'ok',
'data': {
'inventory': options['inventory'],
'options': options,
'name': name,
'entry': entry_record['name'],
'book_id': book_id,
'book_name': book_record['name'],
'roles': role_names,
'inventory_type': inventory_type,
'private_key': private_key,
'template': template,
'extra': extra,
'status': status,
}
}
def job_detail(_id):
query = request.args
record = Job.find_by_id(_id)
if not record:
return jsonify({
'message': 'record not found',
'code': 104040,
}), 404
if record.get('type') == 'adhoc':
template = record.get('template')
inventory = template.get('inventory')
inventory_content = parse_cmdb_inventory(inventory)
template['inventory_content'] = inventory_content
else:
book = Book.find_by_id(record.get('book_id'))
record['book_name'] = book.get('name')
template = record.get('template')
result = load_ansible_playbook(record)
if result['message'] == 'ok':
record['command'] = to_playbook_command(result)
command = to_playbook_command(result)
else:
record['command'] = 'invalid ansible command~!'
if template:
app_id = template.get('app')
if app_id:
app = db.collection('apps').find_one({'_id': ObjectId(app_id)})
if app:
template['app_name'] = app.get('name')
template['app_params'] = app.get('params')
inventory_type = template.get('inventory_type')
inventory = template.get('inventory')
if inventory_type == 'file':
inventory_content = parse_file_inventory(inventory)
else:
inventory_content = parse_cmdb_inventory(inventory)
template['inventory_content'] = inventory_content
role_ids = template.get('roles')
if role_ids:
roles = Playbook.find_by_ids(role_ids)
template['roles'] = list(map(lambda i: i.get('name'), roles))
record['template'] = template
page = int(query.get('page', 1))
size = int(query.get('pageSize', 20))
offset = (page - 1) * size
tasks = get_tasks_by_job(_id, offset=offset, limit=size)
tasks = list(tasks)
logs = []
sort = [('_id', -1)]
task = db.collection('tasks').find_one({'job_id': _id}, sort=sort)
if task:
records = db.collection('task_logs').find(
{'task_id': str(task['_id'])})
for item in records:
logs.append({
'content': str(item.get('content')),
'created_at': item.get('created_at')
})
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'job': record,
'logs': logs,
'tasks': tasks,
}
})
def import_book_from_dir(self,
home_path,
book_id,
exclude=None,
links=False,
prefix='/'):
"""
import dir file to db
@todo
"""
book_id = str(book_id)
exclude = exclude or ['*.retry']
bucket = []
cursor = 0
home_path = home_path.rstrip('/')
parent = home_path
book_record = Book.find_by_id(book_id)
model = Model.build_model('playbook')
playbooks = model.find({'book_id': book_id})
paths = map(lambda i: i['path'], playbooks)
paths = list(paths)
pattern = '|'.join(exclude).replace('*', '.*?')
home_path = '/'.join([home_path, ''])
for current, dirs, files in os.walk(home_path,
topdown=True,
followlinks=links):
pathname = current.replace(home_path, '')
if pathname != '/':
pathname = os.path.join(prefix, pathname)
if exclude:
match = re.search(pattern, pathname)
if match:
continue
if pathname in paths:
index = paths.index(pathname)
paths.pop(index)
dir_record = {
'book_id': str(book_record.get('_id')),
'path': pathname,
'is_dir': True,
'is_edit': False,
'seq_no': cursor,
'parent': None,
'created_at': int(time.time()),
}
if not current == home_path:
dir_record['parent'] = parent
meta = get_meta(pathname)
dir_record.update(meta)
dir_record['additions'] = meta
parent = pathname
bucket.append(dir_record)
for file in files:
pathname = parent.rstrip('/') + '/' + file
if exclude:
match = re.match(pattern, pathname)
if match:
continue
cursor += 1
filename = current + '/' + file
can_edit = is_edit(filename)
file_record = dir_record.copy()
file_record['is_edit'] = can_edit
file_record['path'] = pathname
file_record['parent'] = parent
file_record['is_dir'] = False
file_record['seq_no'] = cursor
if is_edit:
with open(filename, 'r', encoding='utf-8') as fd:
file_record['content'] = fd.read()
file_record['md5'] = md5(file_record['content'])
file_record['is_encrypt'] = Vault.is_encrypted(
file_record['content'])
meta = get_meta(file_record['path'])
file_record['additions'] = meta
file_record.update(meta)
bucket.append(file_record)
cursor += 1
is_entry = filter(lambda i: i.get('role') == 'entry', bucket)
is_entry = list(is_entry)
# if not entry set book status to disable
if not is_entry:
Book.update_one({'_id': ObjectId(book_id)},
{'$set': {
'status': 0
}})
for path in paths:
model.delete_one({'book_id': book_id, 'path': path})
mapping = {}
map(lambda i: {mapping['path']: i}, playbooks)
for item in bucket:
record = mapping.get(item['path'])
if not record:
model.insert_one(item)
continue
else:
# inherit old additions
if record['additions']:
item['additions'].update(record['additions'])
model.update_one({'_id': record['_id']}, {'$set': item})
return bucket
