def get_task_history():
query = request.args or {}
page = int(query.get('page', 1))
size = int(query.get('pageSize', 50))
skip = (page - 1) * size
keyword = query.get('keyword')
where = {}
if keyword:
where['name'] = keyword
cursor = db.collection('tasks').find(where, skip=skip, limit=size)
total = cursor.count()
tasks = []
job = Job()
for task in cursor:
job_id = task.get('job_id')
if not job_id:
continue
task['job'] = job.find_by_id(job_id)
tasks.append(task)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'list': tasks,
'total': total,
'page': page,
'pageSize': size,
}
})
def gitlab_job(token):
payload = request.get_json()
object_type = payload.get('object_type')
tag = payload.get('tag')
job_id = payload.get('job_id')
project_id = payload.get('project_id')
job_status = payload.get('job_status')
repository = payload.get('repository')
record = Job().collection.find_one({'token': token})
if not record:
return jsonify({'message': 'invalid token', 'code': 104010}), 401
if object_type != 'job':
return jsonify({
'message': 'only job event allow',
'code': 104031
}), 403
if not job_id or not project_id or job_status != 'success':
return jsonify({'message': 'invalid params', 'code': 104000}), 400
if not tag:
return jsonify({'message': 'only tag allow', 'code': 104032}), 403
app_info = db.collection('apps').find_one(
{'_id': ObjectId(record.get('app_id'))})
if not app_info:
return jsonify({
'message': 'job must be bind with gitlab app',
'code': 104002
}), 400
params = app_info.get('params') or {}
if params.get('project_id') != project_id:
return jsonify({'message': 'illegal project', 'code': 104003}), 400
# if params.get('extract') == 'artifacts':
# wk = Workspace()
# filename = wk.get_gitlab_artifacts_file(record.get('name'), project_id, job_id)
# gitlab = GitlabApi().dowload_artifact(project_id, job_id, filename)
#
# if params.get('extract') == 'docker':
# pass
integration = Integration(app_info.get('type'), params)
integration.install()
ansible_params = load_ansible_playbook(record)
if ansible_params.get('message') is not 'ok':
return jsonify(payload), 400
data = ansible_params.get('data')
wk = Workspace()
res = wk.load_book_from_db(name=data.get('book_name'),
roles=data.get('roles'))
if not res:
return jsonify({
'message': 'load book failed',
'code': 104000,
}), 400
return jsonify({'message': 'ok', 'code': 0, 'data': 1})
def get_jobs():
username = login_user.get('username')
query = request.args
page = int(query.get('page', 1))
size = int(query.get('pageSize', 50))
offset = (page - 1) * size
is_admin = login_user.get('is_admin')
keyword = query.get('keyword')
status = query.get('status')
job_type = query.get('type')
start = query.get('start')
end = query.get('end')
where = {'status': {'$gt': -1}}
if not is_admin:
where['maintainer'] = {'$in': [username]}
if keyword:
where['name'] = {'$regex': keyword}
if status is not None:
where['status'] = status
if job_type:
where['type'] = job_type
date = []
if start:
date.append({
'created_at': {
'$gte': int(time.mktime(time.strptime(start, '%Y-%m-%d')))
}
})
if end:
date.append({
'created_at': {
'$lte': int(time.mktime(time.strptime(end, '%Y-%m-%d')))
}
})
if date:
where['$and'] = date
jobs = Job().collection.find(where, skip=offset, limit=size)
total = jobs.count()
jobs = list(jobs)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'list': jobs,
'total': total,
'page': page,
'pageSize': size,
},
})
def delete_job(_id):
is_admin = login_user.get('is_admin')
record = Job.find_by_id(ObjectId(_id))
if not record:
return jsonify({'message': 'record not found', 'code': 104040}), 404
if not is_admin or login_user.get('username') not in record['maintainer']:
return jsonify({'message': 'permission deny', 'code': 104031}), 403
Job.delete_one({'_id': record['_id']})
return jsonify({
'message': 'ok',
'code': 0,
})
def delete_job(_id):
record = Job.find_by_id(ObjectId(_id))
if not record:
return jsonify({'message': 'record not found', 'code': 104040}), 404
update = {'$set': {'status': -1}}
Job.update_one({'_id': record['_id']}, update=update)
extra = {'record': record}
logger.info('delete job', extra=extra)
return jsonify({
'message': 'ok',
'code': 0,
})
def run_job(_id, history_id=None, **kwargs):
db = Mongo()
record = Job.find_by_id(_id)
if not record:
return False
request_id = str(current_request_id())
username = None if not login_user else login_user.get('username')
params = (_id, request_id, username, history_id)
queue_name = get_queue_by_job(_id)
extra = record.get('extra')
template = record.get('template')
schedule = extra.get('schedule')
ansible_type = record.get('type')
if template.get('run_type') == 'schedule':
existed = db.collection('scheduler_jobs').find_one(
{'_id': record['_id']})
if existed:
return False
scheduler.add_job(func=run_schedule_task,
trigger='cron',
args=params,
coalesce=True,
kwargs=kwargs,
id=str(record.get('_id')),
max_instances=1,
name=record.get('name'),
**schedule)
return True
else:
func = run_playbook_task if ansible_type != 'adhoc' else run_adhoc_task
task = Task(tiger,
func=func,
args=params,
kwargs=kwargs,
queue=queue_name,
unique=True,
lock=True,
lock_key=_id)
task_record = {
'job_id': _id,
'type': 'trigger',
'ansible': ansible_type,
'state': QUEUED,
'queue': queue_name,
'result': '',
'request_id': request_id,
't_id': task.id,
'created_at': time(),
'kwargs': kwargs,
}
result = db.collection('tasks').insert_one(task_record)
task.delay()
return result.inserted_id
def test_get_job(self, check_book, parse_file_inventory,
parse_cmdb_inventory):
data = self.get_data('playbook_job')
data['name'] = str(uuid.uuid4())
data['status'] = 1
url = self.get_api_path('/jobs/%s' % str(ObjectId()))
response = self.client.get(url, headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 154001)
result = self.add_test_data(Job, data)
url = self.get_api_path('/jobs/%s' % str(result.inserted_id))
parse_cmdb_inventory.return_value = ''
parse_file_inventory.return_value = ''
response = self.client.get(url, headers=self.jwt_headers)
self.assert200(response)
Job.update_one({'_id': result.inserted_id}, {
'$set': {
'maintainer': [self.user.get('username')],
'book_id': str(ObjectId())
}
})
response = self.client.get(url, headers=self.jwt_headers)
parse_file_inventory.assert_called()
check_book.assert_called()
self.assert200(response)
self.assertResponseDataHasKey(response, 'logs')
self.assertResponseDataHasKey(response, 'previewContent')
self.assertResponseDataHasKey(response, 'record')
self.assertResponseDataHasKey(response, 'roles')
data = self.get_data('adhoc_job')
data['maintainer'] = [self.user.get('username')]
data['template'] = {
'inventory_type': 'adhoc',
'inventory': 'localhost'
}
result = self.add_test_data(Job, data)
url = self.get_api_path('/jobs/%s' % str(result.inserted_id))
response = self.client.get(url, headers=self.jwt_headers)
parse_cmdb_inventory.assert_called()
self.assertResponseDataHasKey(response, 'logs')
self.assertResponseDataHasKey(response, 'previewContent')
self.assertResponseDataHasKey(response, 'record')
def rollback(_id):
record = db.collection('tasks').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'task not found',
'code': 194041
}), 404
job_id = record.get('job_id')
if not job_id:
return jsonify({
'message': 'invalid job',
'code': 1040011
}), 400
where = {
'_id': ObjectId(job_id),
'status': 1,
}
job_record = Job().collection.find_one(where)
if not job_record:
return jsonify({
'message': 'invalid job',
'code': 1040010
}), 400
history = db.collection('build_history').find_one({'task_id': _id})
if not history:
return jsonify({
'message': 'failed load playbook from history',
'code': 104041
}), 404
build_id = str(history.get('_id'))
run_job(job_id, build_id)
return jsonify({
'message': 'ok',
'code': 0,
})
def check_job(_id):
record = Job.find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({'message': 'job not found', 'code': 104001}), 400
body = {'template': record.get('template'), 'extra': record.get('extra')}
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
return jsonify(payload), 400
data = payload.get('data')
options = data.get('options')
private_key = data.get('private_key')
wk = Workspace()
res = wk.load_book_from_db(name=data.get('book_name'),
roles=data.get('roles'))
if not res:
return jsonify({
'message': 'book not found',
'code': 104000,
}), 400
entry = wk.get_book_entry(data.get('book_name'), data.get('entry'))
with NamedTemporaryFile('w+t', delete=True) as fd:
key_text = get_credential_content_by_id(private_key, 'private_key')
if not key_text:
return jsonify({
'message': 'invalid private_key',
'code': 104033,
}), 401
fd.write(key_text)
fd.seek(0)
options['private_key'] = fd.name
play = PlayBookRunner(data['inventory'], options)
play.run(entry)
result = play.get_result()
return jsonify({'message': 'ok', 'code': 0, 'data': result})
def job_webhook():
query = request.args
token = query.get('token')
payload = request.get_json()
if not payload or not token:
return jsonify({'message': 'invalid params', 'code': 104000}), 400
record = Job().collection.find_one({'token': token})
if not record:
return jsonify({'message': 'illegal token', 'code': 104010}), 401
if record.get('type') == 'adhoc':
task_id = run_job(str(record.get('_id')))
if not task_id:
return jsonify({
'message': 'try to queue task faield',
'code': 104008
}), 400
return jsonify({'message': 'ok', 'code': 0, 'data': task_id})
template = record.get('template')
app_id = template.get('app')
income_params = {'cache': True}
if app_id:
app_info = Application.find_by_id(app_id)
if not app_info:
return jsonify({
'message': 'app not found, please check your app',
'code': 104001
}), 400
app_params = app_info.get('params')
income = app_params.get('income')
if income and payload.get('income'):
income = Template(income)
tpl = income.render(**payload.get('income'))
tpl = yaml.safe_load(tpl)
if tpl:
income_params.update(tpl)
task_id = run_job(str(record.get('_id')), **income_params)
if not task_id:
return jsonify({
'message': 'put job enqueue failed',
'code': 104002
}), 400
# if app_type == 'jenkins':
# build_id = '19'
# job_name = 'upward'
# run_job(_id, job_name, build_id)
# elif app_type == 'gitlab':
# project_id = '13539397'
# job_id = '261939258'
# run_job(_id, project_id, job_id)
# else:
# run_job(_id)
# logger.error('test', extra={'a': {'b': 1}})
return jsonify({'message': 'ok', 'code': 0, 'data': task_id})
def get_job(_id):
username = login_user.get('username')
if not _id:
return jsonify({'message': 'invalid id', 'code': 154000}), 400
job = Job.find_one({
'_id': ObjectId(_id),
'maintainer': {
'$in': [username]
}
})
# @todo job status
if not job:
return jsonify({
'message': 'invalid id',
'code': 154001,
}), 400
template = job.get('template')
inventory_type = template.get('inventory_type')
inventory = template.get('inventory')
if job.get('type') == 'adhoc':
inventory_content = parse_cmdb_inventory(inventory)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
},
})
if inventory_type == 'file':
inventory_content = parse_file_inventory(inventory)
else:
inventory_content = parse_cmdb_inventory(inventory)
check_playbook(job['book_id'])
if inventory_type == 'file':
book = Book.find_one({'_id': ObjectId(job['book_id'])})
if not book:
hosts = []
else:
hosts = get_inventory_by_book(book.get('_id'),
book_name=book.get('name'))
else:
hosts = get_inventory_from_cmdb()
roles = []
condition = {
'book_id': str(job['book_id']),
'role': 'roles',
'is_dir': True
}
parent = Playbook.find_one(condition)
if parent:
where = {
'book_id': job['book_id'],
'is_dir': True,
'parent': parent.get('path')
}
cursor = Playbook.find(where)
roles = list(cursor)
logs = None
task = Task.find_one({'job_id': _id})
if task:
log = db.collection('logs').find_one({'task_id': str(task['_id'])})
if log:
logs = log.get('message')
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'record': job,
'previewContent': inventory_content,
'hosts': hosts,
'roles': roles,
'logs': logs,
},
})
def add_jobs():
"""
add job
TODO(sang)
:return: json
"""
body = request.get_json()
user = login_user
if not body:
return jsonify({
'message': 'miss required params',
'code': 104000,
}), 400
job_type = body.get('type')
if job_type == 'adhoc':
return add_adhoc()
current_id = body.get('currentId')
record = None
if current_id:
record = Job.find_by_id(current_id)
if not record:
return jsonify({'message': 'job not found', 'code': 104044}), 400
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
return jsonify(payload), 400
data = payload.get('data')
options = data.get('options')
is_check = body.get('check')
private_key = data.get('private_key')
wk = Workspace()
res = wk.load_book_from_db(name=data.get('book_name'),
roles=data.get('roles'))
if not res:
return jsonify({
'message': 'book not found',
'code': 104000,
}), 400
entry = wk.get_book_entry(data.get('book_name'), data.get('entry'))
dry_run = bool(is_check)
options['check'] = dry_run
if dry_run:
with NamedTemporaryFile('w+t', delete=True) as fd:
if private_key:
key_text = get_credential_content_by_id(
private_key, 'private_key')
if not key_text:
return jsonify({
'message': 'invalid private_key',
'code': 104033,
}), 401
fd.write(key_text)
fd.seek(0)
options['private_key'] = fd.name
play = PlayBookRunner(data['inventory'],
options,
callback=CallbackModule())
play.run(entry)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'result': play.get_result(),
'options': options
}
})
name = data.get('name')
existed = Job.find_one({'name': name})
if existed and not current_id:
return jsonify({
'message': 'name existed',
'code': 104001,
}), 400
token = str(base64.b64encode(bytes(current_request_id(), 'utf8')), 'utf8')
new_record = {
'name': name,
'type': 'playbook',
'token': token,
'description': data.get('description'),
'book_id': data.get('book_id'),
'template': data.get('template'),
'extra': data.get('extra'),
'entry': data['entry'],
'status': data['status'],
'maintainer': [user.get('username')],
'created_at': int(time.time()),
'updated_at': datetime.datetime.now().isoformat(),
}
if record:
Job.update_one({'_id': record['_id']}, update={'$set': new_record})
else:
Job.insert_one(new_record)
return jsonify({
'message': 'ok',
'code': 0,
})
def test_add_adhoc_job(self, parse_cmdb_inventory,
get_credential_content_by_id, adhoc_runner):
data = self.get_data('adhoc_job')
data['name'] = str(uuid.uuid4())
url = self.get_api_path('/jobs')
response = self.client.post(url, data='{}', headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104000)
params = data.copy()
# None module params should response 400
params['module'] = None
params['type'] = 'adhoc'
current_user = self.user
params['maintainer'] = [current_user.get('username')]
payload = self.body(params)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104002)
# test parse_cmdb_inventory return None
parse_cmdb_inventory.return_value = None
params['module'] = 'ls'
payload = self.body(params)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104004)
# test with check and no private_key condition
params['check'] = True
params['private_key'] = None
parse_cmdb_inventory.return_value = 'localhost'
runner_instance = adhoc_runner.return_value
runner_instance.get_result.return_value = data['name']
payload = self.body(params)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert200(response)
runner_instance.run.assert_called()
# test run with private_key
params['private_key'] = True
get_credential_content_by_id.return_value = None
payload = self.body(params)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104004)
# assume get private_key return correct
get_credential_content_by_id.return_value = 'test-private'
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert200(response)
runner_instance.run.assert_called()
params['check'] = False
payload = self.body(params)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert200(response)
runner_instance.run.assert_called()
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert400(response)
record = Job.find_one({'name': data['name']})
assert record
assert record.get('name') == data.get('name')
params['job_id'] = str(ObjectId())
payload = self.body(params)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert404(response)
self.assertResponseCode(response, 104040)
params['job_id'] = str(record.get('_id'))
payload = self.body(params)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert200(response)
Job().collection.delete_one({'name': data['name']})
def test_add_job(self, load_ansible_playbook, workspace, key_mock, runner):
data = self.get_data('playbook_job')
data['name'] = str(uuid.uuid4())
url = self.get_api_path('/jobs')
response = self.client.post(url, data='{}', headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104000)
params = data.copy()
params['type'] = 'playbook'
payload = self.body(params)
load_ansible_playbook.return_value = {'message': 'test', 'code': 123}
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 123)
options = {}
ansible_payload = {
'message': 'ok',
'code': 0,
'data': {
'inventory': options.get('inventory'),
'options': options,
'name': data.get('name'),
'entry': data['entry'],
'book_id': data.get('book_id'),
'book_name': 'test',
'roles': ['test'],
'inventory_type': 'cmdb',
'private_key': True,
'template': {},
'extra': {},
'status': 1,
}
}
load_ansible_playbook.return_value = ansible_payload
instance = workspace.return_value
instance.load_book_from_db.return_value = False
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104000)
instance.load_book_from_db.return_value = True
instance.get_book_entry.return_value = '/dev/null'
params['check'] = True
params['private_key'] = True
payload = self.body(params)
key_mock.return_value = 'test_private_key'
instance = runner.return_value
instance.get_result.return_value = 'f**k'
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert200(response)
params['check'] = False
payload = self.body(params)
instance.get_result.return_value = 'f**k'
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert200(response)
response = self.client.post(url,
data=payload,
headers=self.jwt_headers)
self.assert400(response)
self.assertResponseCode(response, 104001)
Job().collection.delete_one({'name': data['name']})
def job_detail(_id):
query = request.args
record = Job.find_by_id(_id)
if not record:
return jsonify({
'message': 'record not found',
'code': 104040,
}), 404
if record.get('type') == 'adhoc':
template = record.get('template')
inventory = template.get('inventory')
inventory_content = parse_cmdb_inventory(inventory)
template['inventory_content'] = inventory_content
else:
book = Book.find_by_id(record.get('book_id'))
record['book_name'] = book.get('name')
template = record.get('template')
result = load_ansible_playbook(record)
if result['message'] == 'ok':
record['command'] = to_playbook_command(result)
command = to_playbook_command(result)
else:
record['command'] = 'invalid ansible command~!'
if template:
app_id = template.get('app')
if app_id:
app = db.collection('apps').find_one({'_id': ObjectId(app_id)})
if app:
template['app_name'] = app.get('name')
template['app_params'] = app.get('params')
inventory_type = template.get('inventory_type')
inventory = template.get('inventory')
if inventory_type == 'file':
inventory_content = parse_file_inventory(inventory)
else:
inventory_content = parse_cmdb_inventory(inventory)
template['inventory_content'] = inventory_content
role_ids = template.get('roles')
if role_ids:
roles = Playbook.find_by_ids(role_ids)
template['roles'] = list(map(lambda i: i.get('name'), roles))
record['template'] = template
page = int(query.get('page', 1))
size = int(query.get('pageSize', 20))
offset = (page - 1) * size
tasks = get_tasks_by_job(_id, offset=offset, limit=size)
tasks = list(tasks)
logs = []
sort = [('_id', -1)]
task = db.collection('tasks').find_one({'job_id': _id}, sort=sort)
if task:
records = db.collection('task_logs').find(
{'task_id': str(task['_id'])})
for item in records:
logs.append({
'content': str(item.get('content')),
'created_at': item.get('created_at')
})
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'job': record,
'logs': logs,
'tasks': tasks,
}
})
def run_playbook_task(_id, request_id, username, history_id, **kwargs):
db = Mongo()
record = Job.find_by_id(ObjectId(_id))
task_record = TaskModel.find_one({'request_id': request_id})
if not task_record:
return False
start_at = time()
state = 'progressing'
result = ''
task_id = task_record.get('_id')
job_id = task_record.get('job_id')
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stderr = sys.stdout = temp_stdout = Reporter(str(task_id))
try:
if history_id:
history = db.collection('build_history').find_one(
{'_id': ObjectId(history_id)})
record = history['job_info']
kwargs = task_record.get('kwargs')
template = record.get('template')
body = {
'template': record.get('template'),
'extra': record.get('extra')
}
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
raise Exception('load ansible options error: ' +
payload.get('message'))
app_id = template.get('app')
if app_id:
app_info = Application.find_by_id(ObjectId(app_id))
if not app_info:
raise Exception('app not found: {}'.format(app_id))
app_type = app_info.get('type')
app_params = app_info.get('params')
if kwargs:
app_params.update(kwargs)
integration = Integration(app_type, app_params)
integration.install()
data = payload.get('data')
options = data.get('options')
private_key = data.get('private_key')
wk = Workspace()
roles = data.get('roles')
if history_id:
bookspace = wk.build_book(history_id)
else:
bookname = data.get('book_name')
bookspace = wk.load_book_from_db(name=bookname,
roles=roles,
build_id=task_id)
if not bookspace or not os.path.isdir(bookspace):
raise Exception('install playbook failed, book name: {}'.format(
data.get('book_name')))
entry = os.path.join(bookspace, data.get('entry'))
with NamedTemporaryFile('w+t', delete=False) as fd:
if private_key:
key_text = get_credential_content_by_id(
private_key, 'private_key')
if not key_text:
raise Exception('invalid private_key')
fd.write(key_text)
fd.seek(0)
options['private-key'] = fd.name
options['tags'] = ['uptime']
options['verbosity'] = 2
inventory = data.get('inventory')
logger.info('ansible-playbook run load inventory: \n{}'.format(
yaml.safe_dump(inventory)))
play = PlayBookRunner(data.get('inventory'),
options,
job_id=job_id)
play.run(entry)
result = play.get_result()
builds = db.collection('build_history').count({'job_id': _id})
state = 'finish'
# @todo
if builds > cache_result_numer:
last_one = db.collection('build_history').find_one(
{'job_id': _id}, sort=[('_id', 1)])
if last_one:
db.fs().delete(last_one.get('file_id'))
db.collection('build_history').delete_one(
{'_id': last_one['_id']})
with TemporaryDirectory() as dir_name:
bookname = data.get('book_name')
zip_file = os.path.join(dir_name, bookname)
zip_file = make_zip(bookspace, zip_file)
with open(zip_file, mode='rb') as stream:
filename = bookname + '.zip'
file_id = db.save_file(filename=filename, fileobj=stream)
store_info = {
'task_id': str(task_id),
'file_id': str(file_id),
'job_id': str(_id),
'job_info': record,
'filename': filename,
'created_at': time(),
'kwargs': kwargs,
}
db.collection('build_history').insert_one(store_info)
shutil.rmtree(bookspace)
except Exception as e:
result = str(e)
extra = {'task_id': task_id}
logger.error('run task with exception: {}'.format(str(e)), extra=extra)
state = 'error'
extra_options = record.get('extra')
user = User.find_one({'username': username})
if user:
user_id = str(user['_id'])
notification = extra_options.get('notification')
message = '[error]run job: {}, message: {}'.format(
record.get('name'), str(e))
sys.stdout.write(message)
if notification and type(notification) == list:
Notify().dispatch(user_id=user_id,
msg_type='task',
content=message,
channel=notification)
finally:
content = temp_stdout.getvalue()
temp_stdout.close(True)
sys.stdout = old_stdout
sys.stderr = old_stderr
finish_at = time()
update = {
'$set': {
'start_at': start_at,
'finish_at': finish_at,
'state': state,
'duration': finish_at - start_at,
'result': result,
}
}
TaskModel.update_one({'_id': task_id}, update=update)
trace = {
'task_id': str(task_id),
'request_id': request_id,
'username': username,
'content': str(content),
'created_at': time(),
}
db.collection('task_logs').insert_one(trace)
def add_adhoc():
payload = request.get_json()
if not payload:
return jsonify({
'message': 'invalid params',
'code': 104001,
}), 400
module = payload.get('module')
args = payload.get('args')
inventory = payload.get('inventory')
private_key = payload.get('private_key')
verbosity = payload.get('verbosity', 0)
name = payload.get('name')
schedule = payload.get('schedule')
check = payload.get('check')
job_id = payload.get('job_id')
extra_options = payload.get('extraOptions')
status = int(payload.get('status', 0))
notification = payload.get('notification')
maintainer = payload.get('maintainer') or []
if maintainer and isinstance(maintainer, list):
users = User.find({'username': {'$in': maintainer}})
names = list(map(lambda i: i['username'], users))
maintainer.extend(names)
if login_user.get('username'):
maintainer.append(login_user.get('username'))
if not job_id:
existed = Job.find_one({'name': name})
if existed and existed.get('status') != -1:
return jsonify({'message': 'name exist', 'code': 104007}), 400
if not module or not inventory or not name:
return jsonify({
'message': 'miss required params',
'code': 104002,
}), 400
# check_module = db.collection('ansible_modules').find_one({
# 'name': module
# })
#
# if not check_module:
# return jsonify({
# 'message': 'invalid module',
# 'code': 104003,
# }), 400
inventory_content = parse_cmdb_inventory(inventory)
if not inventory_content:
return jsonify({
'message': 'invalid inventory',
'code': 104004,
}), 400
options = {}
if extra_options:
options.update(extra_options)
if verbosity:
options['verbosity'] = verbosity
if check:
with NamedTemporaryFile('w+t', delete=True) as fd:
if private_key:
key_text = get_credential_content_by_id(
private_key, 'private_key')
if not key_text:
return jsonify({
'message': 'invalid private key',
'code': 104004,
}), 400
fd.write(key_text)
fd.seek(0)
options['private_key'] = fd.name
tasks = [{'action': {'module': module, 'args': args}}]
hosts = inventory_content
runner = AdHocRunner(hosts, options=options)
runner.run('all', tasks)
result = runner.get_result()
return jsonify({'message': 'ok', 'code': 0, 'data': result})
else:
token = str(base64.b64encode(bytes(current_request_id(), 'utf8')),
'utf8')
data = {
'name': name,
'template': {
'inventory': inventory,
'private_key': private_key,
'verbosity': verbosity,
'module': module,
'args': args,
'extraOptions': extra_options,
},
'extra': {
'schedule': schedule,
'notification': notification,
},
'token': token,
'maintainer': maintainer,
'type': 'adhoc',
'created_at': time.time(),
'status': status,
'add_by': login_user.get('username')
}
if job_id:
record = Job.find_one({'_id': ObjectId(job_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040
}), 404
update = {
'$set': data,
}
Job.update_one({'_id': ObjectId(job_id)}, update=update)
else:
Job.insert_one(data)
return jsonify({
'message': 'ok',
'code': 0,
})
