def get_total_counts_grouped_by_categories(self, job_id):
dataset = {}
pipeline = [{
"$match": {
"job_id": job_id
}
}, {
"$group": {
"_id": "$category",
"count": {
"$sum": 1
}
}
}]
cursor = self.log_collection.aggregate(pipeline, cursor={})
for item in cursor:
key = item["_id"]
value = item["count"]
dataset[key] = value
if not dataset:
raise RecordNotFoundException(
"Record is not found by the job id '%s'" % job_id)
logger.info("DB Level-Total counts grouped by categories are:%s" %
str(dataset))
return dataset
def inventory_collection():
"""
The POST data should be in this format:
{
"name": <inventory name>,
"content": <inventory content>
}
:return:
{
"name": <inventory name>,
"content": <inventory content>,
"path": <inventory path on the server>,
"message": "created"
}
"""
try:
if request.method == 'GET':
logger.info('hi')
return Response('{ "msg":"hello"}', mimetype='application/json')
except Exception, e:
logger.error("Error %s" % str(e), exc_info=True)
return Response('error', mimetype='application/json')
def inventory_collection():
"""
The POST data should be in this format:
{
"name": <inventory name>,
"content": <inventory content>
}
:return:
{
"name": <inventory name>,
"content": <inventory content>,
"path": <inventory path on the server>,
"message": "created"
}
"""
try:
if request.method == 'GET':
logger.info('hi')
return Response('{ "msg":"hello"}', mimetype='application/json')
except Exception, e:
logger.error("Error %s" % str(e), exc_info=True)
return Response('error', mimetype='application/json')
def get_job_details(self, job_id, projection_dict=None):
job_doc = self.job_collection.find_one({"_id": ObjectId(job_id)}, projection=projection_dict)
if not job_doc:
raise RecordNotFoundException("Record is not found by the job id '%s'" % job_id)
job_doc["_id"] = str(job_doc.get("_id"))
logger.info("Job is:%s" % job_doc)
return job_doc
def __init__(self):
logger.info("Initializing 'JobPersistence' object.")
self.job_collection = job_db["ansible_job"]
LOG_COLLECTION_NAME = "ansible_log"
collection_name_list = job_db.collection_names(include_system_collections=False)
if LOG_COLLECTION_NAME not in collection_name_list:
job_db.create_collection(LOG_COLLECTION_NAME, size=1 * 1024 * 1024 * 1024, capped=True)
self.log_collection = job_db[LOG_COLLECTION_NAME]
def get_job_details(self, job_id, projection_dict=None):
job_doc = self.job_collection.find_one({"_id": ObjectId(job_id)},
projection=projection_dict)
if not job_doc:
raise RecordNotFoundException(
"Record is not found by the job id '%s'" % job_id)
job_doc["_id"] = str(job_doc.get("_id"))
logger.info("Job is:%s" % job_doc)
return job_doc
def get_logs_output(job_id):
'''
Get the logs based on the job_id.
To call this API, the form is:
http://<host>:<port>/logs/<job_id>?read_size=<positive integer>
Method: Get
The client should periodically call this Rest API during the specified job's execution.
It returns part of job's logs. The response would be:
1. If the logs file isn't generated when the client calls this API, empty json body "{}" returns.
2. Parts of the job's logs. json body is like:
{
"read_size": 10000,
"job_status": "Executing",
"current_log_content": "\'PLAY RECAP:*****************\n\'Task Name:"
}
The "read_size" means how many bytes the client has read until last invocation. It's incremental value based on the growing file size.
It's mandatory to help this API to locate the logs file to know where to read the rest parts this time.
When calling this API in the first time, the read_size should be 0.
The returned "job_status" tells the client to decide when to stop calling this API.
The "job_status" includes "Success", "Failure", "New" and "Executing".
The client should stop calling this API when the status is "Success" or "Failure".
The "current_log_content" contains parts of the logs output.
'''
logger.info("Enter")
response = None
try:
read_size = request.args.get("read_size", None)
logger.info("read_size is : %s" % read_size)
if (read_size is None):
raise MissingKeyException(
"'read_size' query parameter is required!")
if not read_size.isdigit():
raise InvalidDataTypeException(
"The value of 'read_size' must be either 0 or positive integer."
)
else:
read_size = int(read_size)
if read_size < 0:
raise InvalidDataException(
"The value of 'read_size' must be greater than or equal to 0.")
log_doc = job_service.get_logs_output(job_id, read_size)
response = make_response(json.jsonify(log_doc), 200)
except (MissingKeyException, InvalidDataTypeException,
InvalidDataException), e:
response = _make_error_response(400, str(e))
def get_job_list(self, page_index, page_size):
skip_item = (page_index - 1) * page_size
results = self.job_collection.find(skip=skip_item, limit=page_size)
job_list = []
job_list_dict = {}
for job in results:
job["_id"] = str(job.get("_id"))
job_list.append(job)
logger.info("Appending to job_list:%s" % str(job))
logger.info("Job list size is:%d" % len(job_list))
job_list_dict["job_list"] = job_list
return job_list_dict
def get_job_list(self, page_index, page_size):
skip_item = (page_index - 1) * page_size
results = self.job_collection.find(skip=skip_item, limit=page_size)
job_list = []
job_list_dict = {}
for job in results:
job["_id"] = str(job.get("_id"))
job_list.append(job)
logger.info("Appending to job_list:%s" % str(job))
logger.info("Job list size is:%d" % len(job_list))
job_list_dict["job_list"] = job_list
return job_list_dict
def _create_log_output_file_path(job_id):
current_file_realpath = os.path.realpath(__file__)
current_file_dir = os.path.dirname(current_file_realpath)
project_root_dir = os.path.dirname(current_file_dir)
log_output_file_dir_realpath = project_root_dir + LOG_OUTPUT_RELATIVE_DIR
if not os.path.exists(log_output_file_dir_realpath):
os.mkdir(log_output_file_dir_realpath)
log_output_file = "/%s.log" % job_id
log_output_file_realpath = log_output_file_dir_realpath + log_output_file
logger.info("log output file real path is: %s" % log_output_file_realpath)
return log_output_file_realpath
def _create_log_output_file_path(job_id):
current_file_realpath = os.path.realpath(__file__)
current_file_dir = os.path.dirname(current_file_realpath)
project_root_dir = os.path.dirname(current_file_dir)
log_output_file_dir_realpath = project_root_dir + LOG_OUTPUT_RELATIVE_DIR
if not os.path.exists(log_output_file_dir_realpath):
os.mkdir(log_output_file_dir_realpath)
log_output_file = "/%s.log" % job_id
log_output_file_realpath = log_output_file_dir_realpath + log_output_file
logger.info("log output file real path is: %s" % log_output_file_realpath)
return log_output_file_realpath
def __init__(self):
logger.info("Initializing 'JobPersistence' object.")
self.job_collection = job_db["ansible_job"]
LOG_COLLECTION_NAME = "ansible_log"
collection_name_list = job_db.collection_names(
include_system_collections=False)
if LOG_COLLECTION_NAME not in collection_name_list:
job_db.create_collection(LOG_COLLECTION_NAME,
size=1 * 1024 * 1024 * 1024,
capped=True)
self.log_collection = job_db[LOG_COLLECTION_NAME]
def get_logs_output(job_id):
'''
Get the logs based on the job_id.
To call this API, the form is:
http://<host>:<port>/logs/<job_id>?read_size=<positive integer>
Method: Get
The client should periodically call this Rest API during the specified job's execution.
It returns part of job's logs. The response would be:
1. If the logs file isn't generated when the client calls this API, empty json body "{}" returns.
2. Parts of the job's logs. json body is like:
{
"read_size": 10000,
"job_status": "Executing",
"current_log_content": "\'PLAY RECAP:*****************\n\'Task Name:"
}
The "read_size" means how many bytes the client has read until last invocation. It's incremental value based on the growing file size.
It's mandatory to help this API to locate the logs file to know where to read the rest parts this time.
When calling this API in the first time, the read_size should be 0.
The returned "job_status" tells the client to decide when to stop calling this API.
The "job_status" includes "Success", "Failure", "New" and "Executing".
The client should stop calling this API when the status is "Success" or "Failure".
The "current_log_content" contains parts of the logs output.
'''
logger.info("Enter")
response = None
try:
read_size = request.args.get("read_size", None)
logger.info("read_size is : %s" % read_size)
if (read_size is None):
raise MissingKeyException("'read_size' query parameter is required!")
if not read_size.isdigit():
raise InvalidDataTypeException("The value of 'read_size' must be either 0 or positive integer.")
else:
read_size = int(read_size)
if read_size < 0:
raise InvalidDataException(
"The value of 'read_size' must be greater than or equal to 0.")
log_doc = job_service.get_logs_output(job_id, read_size)
response = make_response(json.jsonify(log_doc), 200)
except (MissingKeyException, InvalidDataTypeException, InvalidDataException), e:
response = _make_error_response(400, str(e))
def get_total_counts_grouped_by_categories(self, job_id):
dataset = {}
pipeline = [{"$match": {"job_id": job_id}},
{"$group": {"_id": "$category", "count": {"$sum": 1}}}
]
cursor = self.log_collection.aggregate(pipeline, cursor={})
for item in cursor:
key = item["_id"]
value = item["count"]
dataset[key] = value
if not dataset:
raise RecordNotFoundException("Record is not found by the job id '%s'" % job_id)
logger.info("DB Level-Total counts grouped by categories are:%s" % str(dataset))
return dataset
def get_job_details(job_id):
'''
Retrieve the job details.
To call this API, the form is:
http://<host>:<port>/jobs/<job_id>
Method: Get
'''
logger.info("Enter")
response = None
try:
if not job_id:
raise MissingDataException("Missing 'job_id'.")
job_doc = job_service.get_job_details(job_id)
response = make_response(json.jsonify(job_doc), 200)
except (MissingDataException, RecordNotFoundException), e:
response = _make_error_response(404, str(e))
def get_job_details(job_id):
'''
Retrieve the job details.
To call this API, the form is:
http://<host>:<port>/jobs/<job_id>
Method: Get
'''
logger.info("Enter")
response = None
try:
if not job_id:
raise MissingDataException("Missing 'job_id'.")
job_doc = job_service.get_job_details(job_id)
response = make_response(json.jsonify(job_doc), 200)
except (MissingDataException, RecordNotFoundException), e:
response = _make_error_response(404, str(e))
def get_task_duration_grouped_by_names(job_id):
'''
Get each task duration which is associated with 'job_id' and grouped by task names.
A task in playbook could be executed multiple times, so the task duration is an accumulated value.
The value of the duration is seconds.
This API intends to be used by client for drawing the chart.
To call this API, the form is:
http://<host>:<port>/logs/task-duration-grouped-by-names/<job_id>
Method: Get
Response body example:
{
task_duration_list:[
{'Install MongoDB' : 1.55555555},
{'Show print message' : 0.00111}
]
}
'''
logger.info("Enter")
response = None
try:
size = request.args.get("size", None)
logger.info("size is : %s" % size)
if size is None:
raise MissingKeyException("The 'size' field is required.")
if not size.isdigit():
raise InvalidDataTypeException(
"The 'size' must be positive integer.")
else:
size = int(size)
if size <= 0:
raise InvalidDataException("The 'size' must be greater than 0.")
task_dict = job_service.get_task_duration_grouped_by_names(
job_id, size)
response = make_response(json.jsonify(task_dict), 200)
except (MissingKeyException, InvalidDataTypeException,
InvalidDataException), e:
response = _make_error_response(400, str(e))
def get_job_status(job_id):
'''
Retrieve the job status.
The client should periodically call this api to get the latest job status.
Job status includes: 'New', 'Executing', 'Failure' and 'Success'.
To call this API, the form is:
http://<host>:<port>/jobs/job-status/<job_id>
Method: Get
It returns a 'status' corresponding to above job_id.
'''
logger.info("Enter")
response = None
try:
if not job_id:
raise MissingDataException("Missing 'job_id'.")
job_doc = job_service.get_job_details(job_id, {"_id": True, "status": True})
response = make_response(json.jsonify(job_doc))
except (MissingDataException, RecordNotFoundException), e:
response = _make_error_response(404, str(e))
def get_task_duration_grouped_by_names(job_id):
'''
Get each task duration which is associated with 'job_id' and grouped by task names.
A task in playbook could be executed multiple times, so the task duration is an accumulated value.
The value of the duration is seconds.
This API intends to be used by client for drawing the chart.
To call this API, the form is:
http://<host>:<port>/logs/task-duration-grouped-by-names/<job_id>
Method: Get
Response body example:
{
task_duration_list:[
{'Install MongoDB' : 1.55555555},
{'Show print message' : 0.00111}
]
}
'''
logger.info("Enter")
response = None
try:
size = request.args.get("size", None)
logger.info("size is : %s" % size)
if size is None:
raise MissingKeyException("The 'size' field is required.")
if not size.isdigit():
raise InvalidDataTypeException("The 'size' must be positive integer.")
else:
size = int(size)
if size <= 0:
raise InvalidDataException("The 'size' must be greater than 0.")
task_dict = job_service.get_task_duration_grouped_by_names(job_id, size)
response = make_response(json.jsonify(task_dict), 200)
except (MissingKeyException, InvalidDataTypeException, InvalidDataException), e:
response = _make_error_response(400, str(e))
def get_task_duration_grouped_by_names(self, job_id, size):
data_list = []
dataset = {}
pipeline = [
{"$match": {"job_id": job_id}},
{"$group": {"_id": "$task_name", "duration": {"$sum": "$task_duration"}}},
{"$sort": {"duration": -1}}
]
log_set = self.log_collection.aggregate(pipeline, cursor={})
i = 1
for item in log_set:
if i > size:
break
key = item["_id"]
value = item["duration"]
data_list.append({key: value})
i = i + 1
if not data_list:
raise RecordNotFoundException("Record is not found by the job id '%s'" % job_id)
dataset["task_duration_list"] = data_list
logger.info("DB Level-Total task duration grouped by task names are:%s" % str(dataset))
return dataset
def get_task_duration_grouped_by_names(self, job_id, size):
data_list = []
dataset = {}
pipeline = [{
"$match": {
"job_id": job_id
}
}, {
"$group": {
"_id": "$task_name",
"duration": {
"$sum": "$task_duration"
}
}
}, {
"$sort": {
"duration": -1
}
}]
log_set = self.log_collection.aggregate(pipeline, cursor={})
i = 1
for item in log_set:
if i > size:
break
key = item["_id"]
value = item["duration"]
data_list.append({key: value})
i = i + 1
if not data_list:
raise RecordNotFoundException(
"Record is not found by the job id '%s'" % job_id)
dataset["task_duration_list"] = data_list
logger.info(
"DB Level-Total task duration grouped by task names are:%s" %
str(dataset))
return dataset
def get_job_status(job_id):
'''
Retrieve the job status.
The client should periodically call this api to get the latest job status.
Job status includes: 'New', 'Executing', 'Failure' and 'Success'.
To call this API, the form is:
http://<host>:<port>/jobs/job-status/<job_id>
Method: Get
It returns a 'status' corresponding to above job_id.
'''
logger.info("Enter")
response = None
try:
if not job_id:
raise MissingDataException("Missing 'job_id'.")
job_doc = job_service.get_job_details(job_id, {
"_id": True,
"status": True
})
response = make_response(json.jsonify(job_doc))
except (MissingDataException, RecordNotFoundException), e:
response = _make_error_response(404, str(e))
def get_total_counts_grouped_by_categories(job_id):
'''
Get the total counts of ansible task logs that are associated with 'job_id' and grouped by categories.
The categories of ansible task are 'FAILED', 'OK' and 'UNREACHABLE'.
This API intends to be used by client for drawing the chart.
To call this API, the form is:
http://<host>:<port>/logs/total-counts-grouped-by-categories/<job_id>
Method: Get
Response body example:
{
'FAILED' : 2,
'OK' : 10,
'UNREACHABLE' : 0
}
'''
logger.info("Enter")
response = None
try:
count_dict = job_service.get_total_counts_grouped_by_categories(job_id)
response = make_response(json.jsonify(count_dict), 200)
except RecordNotFoundException, e:
response = _make_error_response(404, str(e))
def get_total_counts_grouped_by_categories(job_id):
'''
Get the total counts of ansible task logs that are associated with 'job_id' and grouped by categories.
The categories of ansible task are 'FAILED', 'OK' and 'UNREACHABLE'.
This API intends to be used by client for drawing the chart.
To call this API, the form is:
http://<host>:<port>/logs/total-counts-grouped-by-categories/<job_id>
Method: Get
Response body example:
{
'FAILED' : 2,
'OK' : 10,
'UNREACHABLE' : 0
}
'''
logger.info("Enter")
response = None
try:
count_dict = job_service.get_total_counts_grouped_by_categories(job_id)
response = make_response(json.jsonify(count_dict), 200)
except RecordNotFoundException, e:
response = _make_error_response(404, str(e))
def __make_ansible_command_str(self, job_doc):
logger.info("Enter")
command_separator = " "
command_list = []
command_list.append("/usr/bin/ansible-playbook")
password = None
become_password = None
inventory_file = job_doc.get("inventory")
playbook_file = job_doc.get("playbook")
forks = job_doc.get("forks")
job_id = str(job_doc.get("_id"))
credentials = job_doc.get("credentials", None)
if credentials:
username = credentials.get("username", None)
if username:
command_list.append("--user=%s" % username)
private_key_file = credentials.get("private_key_file", None)
if private_key_file:
command_list.append("--private-key=%s" % private_key_file)
password = credentials.get("password", None)
if password:
command_list.append("--ask-pass")
become_method = credentials.get("become_method", None)
if become_method:
command_list.append("--become-method=%s" % become_method)
become_user = credentials.get("become_user", None)
if become_user:
command_list.append("--become-user=%s" % become_user)
become_password = credentials.get("become_password", None)
if become_password:
command_list.append("--ask-become-pass")
command_list.append("--inventory-file=%s" % inventory_file)
command_list.append("-e job_id=%s" % job_id)
if forks:
command_list.append("--forks=%d" % forks)
command_list.append(playbook_file)
command_str = command_separator.join(command_list)
logger.info("The ansible command is:%s" % command_str)
logger.info("Exit")
return command_str, password, become_password
def __make_ansible_command_str(self, job_doc):
logger.info("Enter")
command_separator = " "
command_list = []
command_list.append("/usr/bin/ansible-playbook")
password = None
become_password = None
inventory_file = job_doc.get("inventory")
playbook_file = job_doc.get("playbook")
forks = job_doc.get("forks")
job_id = str(job_doc.get("_id"))
credentials = job_doc.get("credentials", None)
if credentials:
username = credentials.get("username", None)
if username:
command_list.append("--user=%s" % username)
private_key_file = credentials.get("private_key_file", None)
if private_key_file:
command_list.append("--private-key=%s" % private_key_file)
password = credentials.get("password", None)
if password:
command_list.append("--ask-pass")
become_method = credentials.get("become_method", None)
if become_method:
command_list.append("--become-method=%s" % become_method)
become_user = credentials.get("become_user", None)
if become_user:
command_list.append("--become-user=%s" % become_user)
become_password = credentials.get("become_password", None)
if become_password:
command_list.append("--ask-become-pass")
command_list.append("--inventory-file=%s" % inventory_file)
command_list.append("-e job_id=%s" % job_id)
if forks:
command_list.append("--forks=%d" % forks)
command_list.append(playbook_file)
command_str = command_separator.join(command_list)
logger.info("The ansible command is:%s" % command_str)
logger.info("Exit")
return command_str, password, become_password
def get_job_list():
'''
Get the job list from the DB
To call this API, the form is:
http://<host>:<port>/jobs?page_index=<positive integer>&page_size=<positive integer>
Method: Get
Both query parameters 'page_index' and 'page_size' must be given and they are positive integers.
'''
logger.info("Enter")
response = None
try:
page_index = request.args.get("page_index", None)
page_size = request.args.get("page_size", None)
logger.info("page_index is : %s" % page_index)
logger.info("page_size is : %s" % page_size)
if (page_index is None) or (page_size is None):
raise MissingKeyException(
"Both 'page_index' and 'page_size' keys are required.")
if (not page_index.isdigit()) or (not page_size.isdigit()):
raise InvalidDataTypeException(
"The values of 'page_index' and 'page_size' must be positive integer."
)
else:
page_index = int(page_index)
page_size = int(page_size)
if (page_index < 1) or (page_size < 1):
raise InvalidDataException(
"Both the values of 'page_index' and 'page_size' must be greater than or equal to 1."
)
job_list = job_service.get_job_list(page_index, page_size)
response = make_response(json.jsonify(job_list), 200)
except (MissingKeyException, InvalidDataTypeException,
InvalidDataException), e:
response = _make_error_response(400, str(e))
def get_job_list():
'''
Get the job list from the DB
To call this API, the form is:
http://<host>:<port>/jobs?page_index=<positive integer>&page_size=<positive integer>
Method: Get
Both query parameters 'page_index' and 'page_size' must be given and they are positive integers.
'''
logger.info("Enter")
response = None
try:
page_index = request.args.get("page_index", None)
page_size = request.args.get("page_size", None)
logger.info("page_index is : %s" % page_index)
logger.info("page_size is : %s" % page_size)
if (page_index is None) or (page_size is None):
raise MissingKeyException("Both 'page_index' and 'page_size' keys are required.")
if (not page_index.isdigit()) or (not page_size.isdigit()):
raise InvalidDataTypeException("The values of 'page_index' and 'page_size' must be positive integer.")
else:
page_index = int(page_index)
page_size = int(page_size)
if (page_index < 1) or (page_size < 1):
raise InvalidDataException(
"Both the values of 'page_index' and 'page_size' must be greater than or equal to 1.")
job_list = job_service.get_job_list(page_index, page_size)
response = make_response(json.jsonify(job_list), 200)
except (MissingKeyException, InvalidDataTypeException, InvalidDataException), e:
response = _make_error_response(400, str(e))
def create_job():
'''
Store a job and launch it. After launching the job, this api returns immediately with a new job id in response.
Meanwhile, the job executes in the backend asynchronously.
To call this API, the form is:
http://<host>:<port>/jobs
Method: POST
Content-Type: application/json
In request body:
'job_name', 'inventory_path' and 'playbook_path' must be given.
'job_description', 'forks' and 'credentials' block are optional.
In 'credentials' block:
if 'username' is given, either 'password' or 'private_key_file' must be given.
if 'password' or 'private_key_file' is not empty, the corresponding 'username' must be given.
'become_method' is optional.
if 'become_user' is given, the 'become_password' must be provided, vice versa.
Request body Example:
{
"name" : "test job 1",
"description" : "test job 1 description.",
"inventory" : "/etc/ansible/hosts",
"playbook" : "/root/ansible_log_test/ansible_test.yml",
"forks" : 5,
"credentials" : {
"username" : "root",
"password" : "MbvG6Nw8",
"become_password" : "MbvG6Nw8",
"become_user" : "root"
}
}
'''
logger.info("Enter")
response = None
try:
_check_content_type("application/json")
logger.info("Request Parameters : %s" % request.data)
request_parameters = request.get_json()
if not isinstance(request_parameters, dict) or not request_parameters:
raise BadRequest
job_name = request_parameters.get("name", None)
if job_name is None:
raise MissingKeyException("The job's 'name' is required.")
if not isinstance(job_name, basestring):
raise InvalidDataTypeException(
"The job's 'name' must be 'String' type.")
if not job_name:
raise InvalidDataException("The job's 'name' must not be empty.")
job_description = request_parameters.get("description", None)
if job_description is not None:
if not isinstance(job_description, basestring):
raise InvalidDataTypeException(
"The job's 'description' must be 'String' type.")
inventory_path = request_parameters.get("inventory", None)
if inventory_path is None:
raise MissingKeyException("The job's 'inventory' is required.")
if not isinstance(inventory_path, basestring):
raise InvalidDataTypeException(
"The job's 'inventory' must be 'String' type.")
if not inventory_path:
raise InvalidDataException(
"The job's 'inventory' must not be empty.")
playbook_path = request_parameters.get("playbook", None)
if playbook_path is None:
raise MissingKeyException("The job's 'playbook' is required.")
if not isinstance(playbook_path, basestring):
raise InvalidDataTypeException(
"The job's 'playbook' must be 'String' type.")
yml_index = playbook_path.find('.yml')
if (not playbook_path) or (yml_index == -1):
raise InvalidDataException(
"The job's 'playbook' must not be empty and must be a valid YAML file path like '<your_path>/test.yml'."
)
credentials = request_parameters.get("credentials", None)
if credentials is not None:
if not isinstance(credentials, dict):
raise InvalidDataTypeException(
"The job's 'credentials' must be 'Object' or 'Dictionary' type."
)
else:
username = credentials.get("username", None)
password = credentials.get("password", None)
private_key_file = credentials.get("private_key_file", None)
if username is not None:
if not isinstance(username, basestring):
raise InvalidDataTypeException(
"The 'username' must be 'String' type.")
if not username:
raise InvalidDataException(
"The 'username' must not be empty string.")
if (password is None) and (private_key_file is None):
raise MissingKeyException(
"Please provide either 'password' or 'private_key_file' corresponding to the 'username' : %s"
% username)
else:
if password is not None:
if not isinstance(password, basestring):
raise InvalidDataTypeException(
"The 'password' must be 'String' type.")
if not password:
raise InvalidDataException(
"The 'password' must not be empty string.")
if private_key_file is not None:
if not isinstance(private_key_file, basestring):
raise InvalidDataTypeException(
"The 'private_key_file' must be 'String' type."
)
if not private_key_file:
raise InvalidDataException(
"The 'private_key_file' must not be empty string."
)
else:
if (password is not None) or (private_key_file
is not None):
raise MissingKeyException(
"please provide 'username' corresponding to the 'password' or 'private_key_file'"
)
become_method = credentials.get("become_method", None)
become_user = credentials.get("become_user", None)
become_password = credentials.get("become_password", None)
if become_method is not None:
if not isinstance(become_method, basestring):
raise InvalidDataTypeException(
"The 'become_method' must be 'String' type.")
if not become_method:
raise InvalidDataException(
"The 'become_method' must not be empty string.")
if become_user is not None:
if not isinstance(become_user, basestring):
raise InvalidDataTypeException(
"The 'become_user' must be 'String' type.")
if not become_user:
raise InvalidDataException(
"The 'become_user' must not be empty string.")
if become_password is None:
raise MissingKeyException(
"Please provide the 'become_password' corresponding to the 'become_user' : %s"
% become_user)
else:
if not isinstance(become_password, basestring):
raise InvalidDataTypeException(
"The 'become_password' must be 'String' type.")
if not become_password:
raise InvalidDataException(
"The 'become_password' must not be empty string."
)
else:
if become_password is not None:
raise MissingKeyException(
"Please provide the 'become_user' corresponding to the 'become_password'."
)
forks = request_parameters.get("forks", None)
if forks is not None:
if not isinstance(forks, int):
raise InvalidDataTypeException("The 'forks' must be integer.")
if (forks < 5):
raise InvalidDataException(
"The 'forks' number must be greater than or equal to 5.")
job_dict = job_service.create_job(request_parameters)
job_service.launch_job(job_dict["_id"])
response = make_response(json.jsonify(job_dict), 201)
except InvalidContentTypeException, e:
response = _make_error_response(415, str(e))
def launch_ansible_playbook_command(ansible_command, job_id, password=None, become_password=None):
logger.info("Enter")
log_output_file_realpath = _create_log_output_file_path(job_id)
output_file = open(log_output_file_realpath, 'a')
try:
if password and become_password:
password_str = password + "\n"
become_password_str = become_password + "\n"
logger.info(pexpect.run(ansible_command, logfile=output_file, timeout=-1,
events={"SSH password:"******"SUDO password.*:": become_password_str}))
elif password:
password_str = password + "\n"
logger.info(
pexpect.run(ansible_command, logfile=output_file, timeout=-1, events={"SSH password:"******"\n"
logger.info(pexpect.run(ansible_command, logfile=output_file, timeout=-1,
events={".*password:"******"Exit")
def get_logs_output(self, job_id, read_size):
logger.info("Enter")
logger.info("log output file's read size is: %u" % read_size)
log_output_file_realpath = _create_log_output_file_path(job_id)
log_output_dict = {}
if os.path.exists(log_output_file_realpath):
output_file_object = open(log_output_file_realpath, 'r')
job_doc = job_DAO.get_job_details(job_id, {"_id": False, "status": True})
job_status = job_doc["status"]
logger.info("Current job status is:%s" % job_status)
current_file_size = os.path.getsize(log_output_file_realpath)
logger.info("Current log output file's size snapshot is:%u" % current_file_size)
unread_size = current_file_size - read_size
logger.info("Current log output file's unread size is:%u" % unread_size)
output_file_object.seek(read_size)
log_content = output_file_object.read(unread_size)
logger.info("Current log output file's content is:%s" % log_content)
log_output_dict["read_size"] = current_file_size
log_output_dict["job_status"] = job_status
log_output_dict["current_log_content"] = log_content
logger.info("Exit")
return log_output_dict
def get_logs_output(self, job_id, read_size):
logger.info("Enter")
logger.info("log output file's read size is: %u" % read_size)
log_output_file_realpath = _create_log_output_file_path(job_id)
log_output_dict = {}
if os.path.exists(log_output_file_realpath):
output_file_object = open(log_output_file_realpath, 'r')
job_doc = job_DAO.get_job_details(job_id, {
"_id": False,
"status": True
})
job_status = job_doc["status"]
logger.info("Current job status is:%s" % job_status)
current_file_size = os.path.getsize(log_output_file_realpath)
logger.info("Current log output file's size snapshot is:%u" %
current_file_size)
unread_size = current_file_size - read_size
logger.info("Current log output file's unread size is:%u" %
unread_size)
output_file_object.seek(read_size)
log_content = output_file_object.read(unread_size)
logger.info("Current log output file's content is:%s" %
log_content)
log_output_dict["read_size"] = current_file_size
log_output_dict["job_status"] = job_status
log_output_dict["current_log_content"] = log_content
logger.info("Exit")
return log_output_dict
def create_job():
'''
Store a job and launch it. After launching the job, this api returns immediately with a new job id in response.
Meanwhile, the job executes in the backend asynchronously.
To call this API, the form is:
http://<host>:<port>/jobs
Method: POST
Content-Type: application/json
In request body:
'job_name', 'inventory_path' and 'playbook_path' must be given.
'job_description', 'forks' and 'credentials' block are optional.
In 'credentials' block:
if 'username' is given, either 'password' or 'private_key_file' must be given.
if 'password' or 'private_key_file' is not empty, the corresponding 'username' must be given.
'become_method' is optional.
if 'become_user' is given, the 'become_password' must be provided, vice versa.
Request body Example:
{
"name" : "test job 1",
"description" : "test job 1 description.",
"inventory" : "/etc/ansible/hosts",
"playbook" : "/root/ansible_log_test/ansible_test.yml",
"forks" : 5,
"credentials" : {
"username" : "root",
"password" : "MbvG6Nw8",
"become_password" : "MbvG6Nw8",
"become_user" : "root"
}
}
'''
logger.info("Enter")
response = None
try:
_check_content_type("application/json")
logger.info("Request Parameters : %s" % request.data)
request_parameters = request.get_json()
if not isinstance(request_parameters, dict) or not request_parameters:
raise BadRequest
job_name = request_parameters.get("name", None)
if job_name is None:
raise MissingKeyException("The job's 'name' is required.")
if not isinstance(job_name, basestring):
raise InvalidDataTypeException("The job's 'name' must be 'String' type.")
if not job_name:
raise InvalidDataException("The job's 'name' must not be empty.")
job_description = request_parameters.get("description", None)
if job_description is not None:
if not isinstance(job_description, basestring):
raise InvalidDataTypeException("The job's 'description' must be 'String' type.")
inventory_path = request_parameters.get("inventory", None)
if inventory_path is None:
raise MissingKeyException("The job's 'inventory' is required.")
if not isinstance(inventory_path, basestring):
raise InvalidDataTypeException("The job's 'inventory' must be 'String' type.")
if not inventory_path:
raise InvalidDataException("The job's 'inventory' must not be empty.")
playbook_path = request_parameters.get("playbook", None)
if playbook_path is None:
raise MissingKeyException("The job's 'playbook' is required.")
if not isinstance(playbook_path, basestring):
raise InvalidDataTypeException("The job's 'playbook' must be 'String' type.")
yml_index = playbook_path.find('.yml')
if (not playbook_path) or (yml_index == -1):
raise InvalidDataException(
"The job's 'playbook' must not be empty and must be a valid YAML file path like '<your_path>/test.yml'.")
credentials = request_parameters.get("credentials", None)
if credentials is not None:
if not isinstance(credentials, dict):
raise InvalidDataTypeException("The job's 'credentials' must be 'Object' or 'Dictionary' type.")
else:
username = credentials.get("username", None)
password = credentials.get("password", None)
private_key_file = credentials.get("private_key_file", None)
if username is not None:
if not isinstance(username, basestring):
raise InvalidDataTypeException("The 'username' must be 'String' type.")
if not username:
raise InvalidDataException("The 'username' must not be empty string.")
if (password is None) and (private_key_file is None):
raise MissingKeyException(
"Please provide either 'password' or 'private_key_file' corresponding to the 'username' : %s" % username)
else:
if password is not None:
if not isinstance(password, basestring):
raise InvalidDataTypeException("The 'password' must be 'String' type.")
if not password:
raise InvalidDataException("The 'password' must not be empty string.")
if private_key_file is not None:
if not isinstance(private_key_file, basestring):
raise InvalidDataTypeException("The 'private_key_file' must be 'String' type.")
if not private_key_file:
raise InvalidDataException("The 'private_key_file' must not be empty string.")
else:
if (password is not None) or (private_key_file is not None):
raise MissingKeyException(
"please provide 'username' corresponding to the 'password' or 'private_key_file'")
become_method = credentials.get("become_method", None)
become_user = credentials.get("become_user", None)
become_password = credentials.get("become_password", None)
if become_method is not None:
if not isinstance(become_method, basestring):
raise InvalidDataTypeException("The 'become_method' must be 'String' type.")
if not become_method:
raise InvalidDataException("The 'become_method' must not be empty string.")
if become_user is not None:
if not isinstance(become_user, basestring):
raise InvalidDataTypeException("The 'become_user' must be 'String' type.")
if not become_user:
raise InvalidDataException("The 'become_user' must not be empty string.")
if become_password is None:
raise MissingKeyException(
"Please provide the 'become_password' corresponding to the 'become_user' : %s" % become_user)
else:
if not isinstance(become_password, basestring):
raise InvalidDataTypeException("The 'become_password' must be 'String' type.")
if not become_password:
raise InvalidDataException("The 'become_password' must not be empty string.")
else:
if become_password is not None:
raise MissingKeyException(
"Please provide the 'become_user' corresponding to the 'become_password'.")
forks = request_parameters.get("forks", None)
if forks is not None:
if not isinstance(forks, int):
raise InvalidDataTypeException("The 'forks' must be integer.")
if (forks < 5):
raise InvalidDataException("The 'forks' number must be greater than or equal to 5.")
job_dict = job_service.create_job(request_parameters)
job_service.launch_job(job_dict["_id"])
response = make_response(json.jsonify(job_dict), 201)
except InvalidContentTypeException, e:
response = _make_error_response(415, str(e))
def launch_ansible_playbook_command(ansible_command,
job_id,
password=None,
become_password=None):
logger.info("Enter")
log_output_file_realpath = _create_log_output_file_path(job_id)
output_file = open(log_output_file_realpath, 'a')
try:
if password and become_password:
password_str = password + "\n"
become_password_str = become_password + "\n"
logger.info(
pexpect.run(ansible_command,
logfile=output_file,
timeout=-1,
events={
"SSH password:"******"SUDO password.*:": become_password_str
}))
elif password:
password_str = password + "\n"
logger.info(
pexpect.run(ansible_command,
logfile=output_file,
timeout=-1,
events={"SSH password:"******"\n"
logger.info(
pexpect.run(ansible_command,
logfile=output_file,
timeout=-1,
events={".*password:"******"Exit")
if (page_index < 1) or (page_size < 1):
raise InvalidDataException(
"Both the values of 'page_index' and 'page_size' must be greater than or equal to 1."
)
job_list = job_service.get_job_list(page_index, page_size)
response = make_response(json.jsonify(job_list), 200)
except (MissingKeyException, InvalidDataTypeException,
InvalidDataException), e:
response = _make_error_response(400, str(e))
except Exception, e:
response = _make_error_response(500, str(e))
logger.info("Exit")
return response
# @api.route("/api/v1/ansible-job/codebase")
# def access_codebase():
# pass
@api.route("/jobs", methods=["POST"])
@cross_origin(origin='*')
def create_job():
'''
Store a job and launch it. After launching the job, this api returns immediately with a new job id in response.
Meanwhile, the job executes in the backend asynchronously.
To call this API, the form is:
page_index = int(page_index)
page_size = int(page_size)
if (page_index < 1) or (page_size < 1):
raise InvalidDataException(
"Both the values of 'page_index' and 'page_size' must be greater than or equal to 1.")
job_list = job_service.get_job_list(page_index, page_size)
response = make_response(json.jsonify(job_list), 200)
except (MissingKeyException, InvalidDataTypeException, InvalidDataException), e:
response = _make_error_response(400, str(e))
except Exception, e:
response = _make_error_response(500, str(e))
logger.info("Exit")
return response
# @api.route("/api/v1/ansible-job/codebase")
# def access_codebase():
# pass
@api.route("/jobs", methods=["POST"])
@cross_origin(origin='*')
def create_job():
'''
Store a job and launch it. After launching the job, this api returns immediately with a new job id in response.
Meanwhile, the job executes in the backend asynchronously.
To call this API, the form is:
