def gitlab_job(token):
payload = request.get_json()
object_type = payload.get('object_type')
tag = payload.get('tag')
job_id = payload.get('job_id')
project_id = payload.get('project_id')
job_status = payload.get('job_status')
repository = payload.get('repository')
record = Job().collection.find_one({'token': token})
if not record:
return jsonify({'message': 'invalid token', 'code': 104010}), 401
if object_type != 'job':
return jsonify({
'message': 'only job event allow',
'code': 104031
}), 403
if not job_id or not project_id or job_status != 'success':
return jsonify({'message': 'invalid params', 'code': 104000}), 400
if not tag:
return jsonify({'message': 'only tag allow', 'code': 104032}), 403
app_info = db.collection('apps').find_one(
{'_id': ObjectId(record.get('app_id'))})
if not app_info:
return jsonify({
'message': 'job must be bind with gitlab app',
'code': 104002
}), 400
params = app_info.get('params') or {}
if params.get('project_id') != project_id:
return jsonify({'message': 'illegal project', 'code': 104003}), 400
# if params.get('extract') == 'artifacts':
# wk = Workspace()
# filename = wk.get_gitlab_artifacts_file(record.get('name'), project_id, job_id)
# gitlab = GitlabApi().dowload_artifact(project_id, job_id, filename)
#
# if params.get('extract') == 'docker':
# pass
integration = Integration(app_info.get('type'), params)
integration.install()
ansible_params = load_ansible_playbook(record)
if ansible_params.get('message') is not 'ok':
return jsonify(payload), 400
data = ansible_params.get('data')
with build_book_from_db(data.get('book_name'),
roles=data.get('roles')) as bookspace:
if not bookspace:
return jsonify({
'message': 'load book failed',
'code': 104000,
}), 400
return jsonify({'message': 'ok', 'code': 0, 'data': 1})
def download_book(_id):
record = Book.find_by_id(_id)
if not record:
return jsonify({'message': 'record not found', 'code': 104040}), 404
name = record.get('name')
wk = Workspace()
with build_book_from_db(name) as bookspace:
filename = name + '.zip'
with NamedTemporaryFile('w+t', delete=False) as fd:
make_zip(bookspace, fd.name)
return send_file(fd.name,
attachment_filename=filename,
as_attachment=True)
def get_tags():
body = request.get_json()
if not body:
return jsonify({
'message': 'miss required params',
'code': 104000,
}), 400
template = body.get('template')
listtags = template.get('listtags')
listtasks = template.get('listtasks')
if not listtags or not listtasks:
return jsonify({
'message': 'invalid params',
'code': 104001,
}), 400
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
return jsonify(payload), 400
data = payload.get('data')
options = data.get('options')
wk = Workspace()
with build_book_from_db(name=data.get('book_name'),
roles=data.get('roles')) as bookspace:
if not bookspace:
return jsonify({
'message': 'book not found',
'code': 104000,
}), 400
entry = wk.get_book_entry(data.get('book_name'), data.get('entry'))
play = PlayBookRunner([data['inventory']], options)
play.run(entry)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'tags': list(play.tags),
'tasks': list(play.tasks),
}
})
def check_job(_id):
record = Job.find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({'message': 'job not found', 'code': 104001}), 400
body = {'template': record.get('template'), 'extra': record.get('extra')}
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
return jsonify(payload), 400
data = payload.get('data')
options = data.get('options')
private_key = data.get('private_key')
wk = Workspace()
res = build_book_from_db(name=data.get('book_name'),
roles=data.get('roles'))
if not res:
return jsonify({
'message': 'book not found',
'code': 104000,
}), 400
entry = wk.get_book_entry(data.get('book_name'), data.get('entry'))
with NamedTemporaryFile('w+t', delete=True) as fd:
key_text = get_credential_content_by_id(private_key, 'private_key')
if not key_text:
return jsonify({
'message': 'invalid private_key',
'code': 104033,
}), 401
fd.write(key_text)
fd.seek(0)
options['private_key'] = fd.name
play = PlayBookRunner(data['inventory'], options)
play.run(entry)
result = play.get_result()
return jsonify({'message': 'ok', 'code': 0, 'data': result})
def run(book_id, run_id, **options):
perform = Perform.find_one({'run_id': run_id})
book = Book.find_by_id(book_id)
start_at = time.time()
state = 'progressing'
result = ''
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stderr = sys.stdout = temp_stdout = Reporter(run_id)
try:
if book.get('repo') == 'git':
vcs = GitDownload(book.get('repo_options'))
install_path = vcs.install()
# @todo
book_name = book.get('name')
with build_book_from_db(book_name, build_id=run_id) as dest:
if not dest:
result = 'install book failed'
logger.warning(result)
state = 'finish'
else:
inventory = options['inventory']
if type(inventory == str):
inventory = os.path.join(dest, inventory)
entry = os.path.join(dest, options['entry'].pop())
if type(options['tags']) == str:
options['tags'] = options['tags'].split(',')
options['basedir'] = dest
print('xxxxxxxxxxxxx????', inventory, options)
runner = PlayBookRunner(inventory, options)
runner.run([entry])
result = runner.get_result()
print('result', result)
state = 'finish'
except Exception as err:
result = str(err)
extra = {'run_id': run_id}
logger.error('run task with exception: {}'.format(result), extra=extra)
state = 'error'
raise err
finally:
content = temp_stdout.getvalue()
temp_stdout.close(True)
sys.stdout = old_stdout
sys.stderr = old_stderr
print(content)
finish_at = time.time()
update = {
'$set': {
'start_at': start_at,
'finish_at': finish_at,
'state': state,
'duration': finish_at - start_at,
'result': str(result),
'trace': content,
}
}
Perform.update_one({'_id': perform['_id']}, update=update)
def lint(book_id, options, config=None):
"""
base on ansiblelint
refer to ansiblelint.__main__.py
:param book_id:
:param options:
:param config:
:return: None
"""
formatter = formatters.Formatter()
options = get_default_options(options)
where = {
'book_id': str(book_id),
'role': 'entry',
}
entries = Playbook.find(where)
if not entries:
return False
book = Book.find_by_id(book_id)
if not book:
return False
if config:
if 'quiet' in config:
options.quiet = options.quiet or config['quiet']
if 'parseable' in config:
options.parseable = options.parseable or config['parseable']
if 'parseable_severity' in config:
options.parseable_severity = options.parseable_severity or \
config['parseable_severity']
if 'use_default_rules' in config:
options.use_default_rules = options.use_default_rules or config['use_default_rules']
if 'verbosity' in config:
options.verbosity = options.verbosity + config['verbosity']
options.exclude_paths.extend(
config.get('exclude_paths', []))
if 'rulesdir' in config:
options.rulesdir = options.rulesdir + config['rulesdir']
if 'skip_list' in config:
options.skip_list = options.skip_list + config['skip_list']
if 'tags' in config:
options.tags = options.tags + config['tags']
if options.quiet:
formatter = formatters.QuietFormatter()
if options.parseable:
formatter = formatters.ParseableFormatter()
if options.parseable_severity:
formatter = formatters.ParseableSeverityFormatter()
# no args triggers auto-detection mode
# if len(args) == 0 and not (options.listrules or options.listtags):
# args = get_playbooks_and_roles(options=options)
if options.use_default_rules:
rulesdirs = options.rulesdir + [default_rulesdir]
else:
rulesdirs = options.rulesdir or [default_rulesdir]
rules = RulesCollection()
for rulesdir in rulesdirs:
rules.extend(RulesCollection.create_from_directory(rulesdir))
if options.listrules:
return 0
if options.listtags:
return 0
if isinstance(options.tags, six.string_types):
options.tags = options.tags.split(',')
skip = set()
for s in options.skip_list:
skip.update(str(s).split(','))
options.skip_list = frozenset(skip)
with build_book_from_db(book.get('name'), options.get('roles')) as book_path:
playbooks = []
for record in entries:
entry = os.path.join(book_path, record['path'][1:])
playbooks.append(entry)
playbooks = sorted(set(playbooks))
matches = list()
checked_files = set()
for playbook in playbooks:
runner = Runner(rules, playbook, options.tags,
options.skip_list, options.exclude_paths,
options.verbosity, checked_files)
matches.extend(runner.run())
matches.sort(key=lambda x: (normpath(x.filename), x.linenumber, x.rule.id))
results = []
for match in matches:
filename = str(match.filename)
filename = filename.replace(book_path, '')
results.append({
'lineNumber': match.linenumber,
'line': str(match.line),
'rule': match.rule.id,
'filename': filename,
'message': match.message,
})
return results
def add_jobs():
"""
add job
TODO(sang)
:return: json
"""
body = request.get_json()
user = login_user
if not body:
return jsonify({
'message': 'miss required params',
'code': 104000,
}), 400
job_type = body.get('type')
if job_type == 'adhoc':
return add_adhoc()
current_id = body.get('currentId')
record = None
if current_id:
record = Job.find_by_id(current_id)
if not record:
return jsonify({'message': 'job not found', 'code': 104044}), 400
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
return jsonify(payload), 400
data = payload.get('data')
options = data.get('options')
is_check = body.get('check')
private_key = data.get('private_key')
wk = Workspace()
book_name = data.get('book_name')
roles = data.get('roles')
with build_book_from_db(name=book_name, roles=roles) as bookspace:
if not bookspace:
return jsonify({'message': 'invalid book', 'code': 104000}), 400
entry = wk.get_book_entry(data.get('book_name'), data.get('entry'))
# entry = os.path.join(bookspace, data['entry'])
dry_run = bool(is_check)
options['check'] = dry_run
if dry_run:
with NamedTemporaryFile('w+t', delete=True) as fd:
if private_key:
key_text = get_credential_content_by_id(
private_key, 'private_key')
if not key_text:
return jsonify({
'message': 'invalid private_key',
'code': 104033,
}), 401
fd.write(key_text)
fd.seek(0)
options['private_key'] = fd.name
play = PlayBookRunner(data['inventory'],
options,
callback=CallbackModule())
play.run(entry)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'result': play.get_result(),
'options': options
}
})
name = data.get('name')
existed = Job.find_one({'name': name})
if existed and not current_id:
return jsonify({
'message': 'name existed',
'code': 104001,
}), 400
token = str(base64.b64encode(bytes(current_request_id(), 'utf8')), 'utf8')
new_record = {
'name': name,
'type': 'playbook',
'token': token,
'description': data.get('description'),
'book_id': data.get('book_id'),
'template': data.get('template'),
'extra': data.get('extra'),
'entry': data['entry'],
'status': data['status'],
'maintainer': [user.get('username')],
'created_at': int(time.time()),
'updated_at': datetime.datetime.now().isoformat(),
}
if record:
Job.update_one({'_id': record['_id']}, update={'$set': new_record})
else:
Job.insert_one(new_record)
return jsonify({
'message': 'ok',
'code': 0,
})
def run_playbook_task(_id, request_id, username, history_id, **kwargs):
db = Mongo()
record = Job.find_by_id(ObjectId(_id))
task_record = TaskModel.find_one({'request_id': request_id})
if not task_record:
return False
start_at = time()
state = 'progressing'
result = ''
task_id = task_record.get('_id')
job_id = task_record.get('job_id')
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stderr = sys.stdout = temp_stdout = Reporter(str(task_id))
try:
if history_id:
history = db.collection('build_history').find_one(
{'_id': ObjectId(history_id)})
record = history['job_info']
kwargs = task_record.get('kwargs')
template = record.get('template')
body = {
'template': record.get('template'),
'extra': record.get('extra')
}
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
raise Exception('load ansible options error: ' +
payload.get('message'))
app_id = template.get('app')
if app_id:
app_info = Application.find_by_id(ObjectId(app_id))
if not app_info:
raise Exception('app not found: {}'.format(app_id))
app_type = app_info.get('type')
app_params = app_info.get('params')
if kwargs:
app_params.update(kwargs)
integration = Integration(app_type, app_params)
integration.install()
data = payload.get('data')
options = data.get('options')
private_key = data.get('private_key')
roles = data.get('roles')
bookname = data.get('book_name')
with build_book_from_db(bookname,
roles=roles,
build_id=task_id,
history_id=history_id) as bookspace:
if not bookspace or not os.path.isdir(bookspace):
raise Exception(
'install playbook failed, book name: {}'.format(
data.get('book_name')))
entry = os.path.join(bookspace, data.get('entry'))
with NamedTemporaryFile('w+t', delete=False) as fd:
if private_key:
key_text = get_credential_content_by_id(
private_key, 'private_key')
if not key_text:
raise Exception('invalid private_key')
fd.write(key_text)
fd.seek(0)
options['private-key'] = fd.name
inventory = data.get('inventory')
# hosts = os.path.join(bookspace, 'hosts.yml')
# fileObject = open(hosts, mode='w+')
# # fileObject.write(inventory)
# yaml.dump_all(inventory, fileObject)
# fileObject.close()
play = PlayBookRunner(inventory, options, job_id=job_id)
play.run(entry)
result = play.get_result()
builds = db.collection('build_history').count({'job_id': _id})
state = 'finish'
# @todo
if builds > cache_result_number:
last_one = db.collection('build_history').find_one(
{'job_id': _id}, sort=[('_id', 1)])
if last_one:
db.fs().delete(last_one.get('file_id'))
db.collection('build_history').delete_one(
{'_id': last_one['_id']})
with TemporaryDirectory() as dir_name:
bookname = data.get('book_name')
zip_file = os.path.join(dir_name, bookname)
zip_file = make_zip(bookspace, zip_file)
with open(zip_file, mode='rb') as stream:
filename = bookname + '.zip'
file_id = db.save_file(filename=filename,
fileobj=stream)
store_info = {
'task_id': str(task_id),
'file_id': str(file_id),
'job_id': str(_id),
'job_info': record.to_dict(),
'filename': filename,
'created_at': time(),
'kwargs': kwargs,
}
db.collection('build_history').insert_one(store_info)
except Exception as e:
result = str(e)
extra = {'task_id': task_id}
logger.error('run task with exception: {}'.format(str(e)), extra=extra)
state = 'error'
extra_options = record.get('extra')
user = User.find_one({'username': username})
if user:
user_id = str(user['_id'])
notification = extra_options.get('notification')
message = '[error]run job: {}, message: {}'.format(
record.get('name'), str(e))
sys.stdout.write(message)
if notification and type(notification) == list:
Notify().dispatch(user_id=user_id,
msg_type='task',
content=message,
channel=notification)
raise e
finally:
content = temp_stdout.getvalue()
temp_stdout.close(True)
sys.stdout = old_stdout
sys.stderr = old_stderr
finish_at = time()
update = {
'$set': {
'start_at': start_at,
'finish_at': finish_at,
'state': state,
'duration': finish_at - start_at,
'result': str(result),
}
}
TaskModel.update_one({'_id': task_id}, update=update)
trace = {
'task_id': str(task_id),
'request_id': request_id,
'username': username,
'content': str(content),
'created_at': time(),
}
db.collection('task_logs').insert_one(trace)