def GetJobList(userName, vcName, jobOwner, num=None):
try:
dataHandler = DataHandler()
jobs = []
hasAccessOnAllJobs = False
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName,
Permission.Collaborator):
hasAccessOnAllJobs = True
if jobOwner != "all" or not hasAccessOnAllJobs:
jobs = jobs + GetUserPendingJobs(userName, vcName)
jobs = jobs + dataHandler.GetJobList(
userName, vcName, num,
"running,queued,scheduling,unapproved,pausing,paused",
("<>", "and"))
else:
jobs = GetUserPendingJobs(jobOwner, vcName)
for job in jobs:
job.pop('jobMeta', None)
dataHandler.Close()
return jobs
except Exception as e:
logger.error('Exception: %s', str(e))
logger.warn("Fail to get job list for user %s, return empty list",
userName)
return []
def RunCommand(command):
dataHandler = DataHandler()
k8sUtils.kubectl_exec("exec %s %s" %
(command["jobId"], command["command"]))
dataHandler.FinishCommand(command["id"])
dataHandler.Close()
return True
def GetJobDetail(userName, jobId):
job = None
dataHandler = DataHandler()
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Collaborator):
job = jobs[0]
job["log"] = ""
#jobParams = json.loads(base64.b64decode(job["jobMeta"]))
#jobPath,workPath,dataPath = GetStoragePath(jobParams["jobPath"],jobParams["workPath"],jobParams["dataPath"])
#localJobPath = os.path.join(config["storage-mount-path"],jobPath)
#logPath = os.path.join(localJobPath,"joblog.txt")
#print logPath
#if os.path.isfile(logPath):
# with open(logPath, 'r') as f:
# log = f.read()
# job["log"] = log
# f.close()
if "jobDescription" in job:
job.pop("jobDescription",None)
try:
log = dataHandler.GetJobTextField(jobId,"jobLog")
try:
if isBase64(log):
log = base64.b64decode(log)
except Exception:
pass
if log is not None:
job["log"] = log
except:
job["log"] = "fail-to-get-logs"
dataHandler.Close()
return job
def GetJobList(userName, num=None):
try:
dataHandler = DataHandler()
jobs = []
if userName != "all":
jobs = jobs + dataHandler.GetJobList(
userName, None, "running,queued,scheduling,unapproved",
("=", "or"))
jobs = jobs + dataHandler.GetJobList(
userName, num, "running,queued,scheduling,unapproved",
("<>", "and"))
else:
jobs = dataHandler.GetJobList(userName, None,
"error,failed,finished,killed",
("<>", "and"))
for job in jobs:
job.pop('jobMeta', None)
dataHandler.Close()
return jobs
except Exception, e:
logger.error('Exception: ' + str(e))
logger.warn("Fail to get job list for user %s, return empty list" %
userName)
return []
def GetJobStatus(jobId):
result = None
dataHandler = DataHandler()
result = dataHandler.GetJobTextFields(jobId,
["jobStatus", "jobTime", "errorMsg"])
dataHandler.Close()
return result
def ListStorages(userName, vcName):
ret = []
dataHandler = DataHandler()
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.User):
ret = dataHandler.ListStorages(vcName)
dataHandler.Close()
return ret
def KillJob(job, desiredState="killed"):
dataHandler = DataHandler()
result, detail = k8sUtils.GetJobStatus(job["jobId"])
dataHandler.UpdateJobTextField(job["jobId"], "jobStatusDetail",
base64.b64encode(json.dumps(detail)))
logging.info("Killing job %s, with status %s, %s" %
(job["jobId"], result, detail))
if "jobDescriptionPath" in job and job["jobDescriptionPath"] is not None:
jobDescriptionPath = os.path.join(config["storage-mount-path"],
job["jobDescriptionPath"])
if os.path.isfile(jobDescriptionPath):
if k8sUtils.kubectl_delete(jobDescriptionPath) == 0:
dataHandler.UpdateJobTextField(job["jobId"], "jobStatus",
desiredState)
return True
else:
dataHandler.UpdateJobTextField(
job["jobId"], "errorMsg",
"Cannot delete job from Kubernetes Cluster!")
else:
dataHandler.UpdateJobTextField(job["jobId"], "errorMsg",
"Cannot find job description file!")
dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
dataHandler.Close()
return False
def delete(self):
args = self.delete_parser.parse_args()
vcName = args["vcName"]
userName = args["userName"]
database = args["database"]
templateName = args["templateName"]
if database == "master":
if AuthorizationManager.HasAccess(userName, ResourceType.Cluster,
"", Permission.Admin):
scope = "master"
else:
return "access denied", 403
elif database == "vc":
if AuthorizationManager.HasAccess(userName, ResourceType.VC,
vcName, Permission.Admin):
scope = "vc:" + vcName
else:
return "access denied", 403
else:
scope = "user:"******"result"] = dataHandler.DeleteTemplate(templateName, scope)
dataHandler.Close()
return generate_response(ret)
def ApproveJob(job):
logging.info("start to Approve job...")
dataHandler = DataHandler()
dataHandler.ApproveJob(job["jobId"])
dataHandler.Close()
return True
def post(self):
args = self.post_parser.parse_args()
vcName = args["vcName"]
userName = args["userName"]
database = args["database"]
templateName = args["templateName"]
if database == "master":
if AuthorizationManager.HasAccess(userName, ResourceType.Cluster,
"", Permission.Admin):
scope = "master"
else:
return "access denied", 403
elif database == "vc":
if AuthorizationManager.HasAccess(userName, ResourceType.VC,
vcName, Permission.Admin):
scope = "vc:" + vcName
else:
return "access denied", 403
else:
scope = "user:"******"Invalid JSON")
dataHandler = DataHandler()
ret = {}
ret["result"] = dataHandler.UpdateTemplate(templateName, scope,
json.dumps(template_json))
dataHandler.Close()
return generate_response(ret)
def GetResourceAcl(resource):
try:
with acl_cache_lock:
res_key = RESOURCE_KEY_PREFIX + resource
return acl_cache[res_key]
except KeyError:
pass
data_handler = None
ret = []
try:
data_handler = DataHandler()
ret = data_handler.GetResourceAcl(resource)
identities = {}
for ace in ret:
id_key = IDENTITY_KEY_PREFIX + ace["identityName"]
if id_key not in identities:
identities[id_key] = []
identities[id_key].append(ace)
with acl_cache_lock:
res_key = RESOURCE_KEY_PREFIX + resource
acl_cache[res_key] = ret
acl_cache.update(identities)
except Exception as e:
logger.error("Failed to get resource acl for %s. Ex: %s", resource,
e)
finally:
if data_handler is not None:
data_handler.Close()
return ret
def cleanup_endpoints():
try:
data_handler = DataHandler()
try:
dead_endpoints = data_handler.GetDeadEndpoints()
for endpoint_id, dead_endpoint in dead_endpoints.items():
print("\n\n\n\n\n\n----------------Begin to cleanup endpoint %s" % endpoint_id)
endpoint_description_path = os.path.join(config["storage-mount-path"], dead_endpoint["endpointDescriptionPath"])
still_running = get_k8s_endpoint(endpoint_description_path)
# empty mean not existing
if still_running == "":
print("Endpoint already gone %s" % endpoint_id)
status = "stopped"
else:
output = k8sUtils.kubectl_delete(endpoint_description_path)
# 0 for success
if output == 0:
status = "stopped"
print("Succeed cleanup endpoint %s" % endpoint_id)
else:
# TODO will need to clean it up eventually
status = "unknown"
print("Clean dead endpoint %s failed, endpoints: %s" % (endpoint_id, dead_endpoint))
dead_endpoint["status"] = status
dead_endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
data_handler.UpdateEndpoint(dead_endpoint)
except Exception as e:
traceback.print_exc()
finally:
data_handler.Close()
except Exception as e:
traceback.print_exc()
def _HasAccess(identity_name, resource_acl_path, permissions):
start_time = time.time()
requested_access = "%s;%s;%s" % (str(identity_name), resource_acl_path,
str(permissions))
value = resource_acl_cache.get(requested_access)
if value is not None:
logger.info('[cached] Yes for %s in time %s' %
(requested_access, time.time() - start_time))
return value
data_handler = None
try:
data_handler = DataHandler()
identities = []
try:
identities = get_identity_info_from_db(data_handler,
identity_name)["groups"]
identities = map(lambda x: int(x), identities)
except Exception as e:
logger.warn("Failed to get identities list: %s" % e)
identities = []
#TODO: handle isDeny
while resource_acl_path:
acl = data_handler.GetResourceAcl(resource_acl_path)
for ace in acl:
ace_id = int(ace["identityId"])
id_in_identities = ace_id in identities
id_in_range = ace_id < INVALID_RANGE_START or ace_id > INVALID_RANGE_END
if ace["identityName"] == identity_name or (
id_in_identities and id_in_range):
permissions = permissions & (~ace["permissions"])
if not permissions:
logger.info(
'Yes for %s in time %s' %
(requested_access, time.time() - start_time))
resource_acl_cache.add(requested_access, True)
return True
resource_acl_path = AuthorizationManager.__GetParentPath(
resource_acl_path)
logger.info("No for %s in time %s" %
(requested_access, time.time() - start_time))
resource_acl_cache.add(requested_access, False)
return False
except Exception as e:
logger.error("Exception: %s" % e)
logger.warn("No (exception) for %s in time %s" %
(requested_access, time.time() - start_time))
return False
finally:
if data_handler is not None:
data_handler.Close()
def GetCommands(userName, jobId):
commands = []
dataHandler = DataHandler()
jobs = dataHandler.GetJob(jobId=jobId)
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Collaborator):
commands = dataHandler.GetCommands(jobId=jobId)
dataHandler.Close()
return commands
def UpdateVC(userName, vcName, quota, metadata):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.IsClusterAdmin(userName):
ret = dataHandler.UpdateVC(vcName, quota, metadata)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def DeleteVC(userName, vcName):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.IsClusterAdmin(userName):
ret = dataHandler.DeleteVC(vcName)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def UpdateStorage(userName, vcName, url, storageType, metadata, defaultMountPath):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.Admin):
ret = dataHandler.UpdateStorage(vcName, url, storageType, metadata, defaultMountPath)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def DeleteStorage(userName, vcName, url):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.Admin):
ret = dataHandler.DeleteStorage(vcName, url)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def AddStorage(userName, vcName, url, storageType, metadata, defaultMountPath):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.IsClusterAdmin(userName):
ret = dataHandler.AddStorage(vcName, url, storageType, metadata, defaultMountPath)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def GetJobStatus(jobId):
result = None
dataHandler = DataHandler()
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
key_list = ["jobStatus", "jobTime", "errorMsg"]
result = {key: jobs[0][key] for key in key_list}
dataHandler.Close()
return result
