def post(self):
args = self.post_parser.parse_args()
vcName = args["vcName"]
userName = args["userName"]
database = args["database"]
templateName = args["templateName"]
if database == "master":
if AuthorizationManager.HasAccess(userName, ResourceType.Cluster,
"", Permission.Admin):
scope = "master"
else:
return "access denied", 403
elif database == "vc":
if AuthorizationManager.HasAccess(userName, ResourceType.VC,
vcName, Permission.Admin):
scope = "vc:" + vcName
else:
return "access denied", 403
else:
scope = "user:"******"Invalid JSON")
dataHandler = DataHandler()
ret = {}
ret["result"] = dataHandler.UpdateTemplate(templateName, scope,
json.dumps(template_json))
dataHandler.Close()
return generate_response(ret)
def delete(self):
args = self.delete_parser.parse_args()
vcName = args["vcName"]
userName = args["userName"]
database = args["database"]
templateName = args["templateName"]
if database == "master":
if AuthorizationManager.HasAccess(userName, ResourceType.Cluster,
"", Permission.Admin):
scope = "master"
else:
return "access denied", 403
elif database == "vc":
if AuthorizationManager.HasAccess(userName, ResourceType.VC,
vcName, Permission.Admin):
scope = "vc:" + vcName
else:
return "access denied", 403
else:
scope = "user:"******"result"] = dataHandler.DeleteTemplate(templateName, scope)
dataHandler.Close()
return generate_response(ret)
def DeleteAce(userName, identityName, resourceType, resourceName):
ret = None
resourceAclPath = AuthorizationManager.GetResourceAclPath(resourceName, resourceType)
if AuthorizationManager.HasAccess(userName, resourceType, resourceName, Permission.Admin):
ret = AuthorizationManager.DeleteAce(identityName, resourceAclPath)
else:
ret = "Access Denied!"
return ret
def ListVCs(userName):
ret = []
vcList = DataManager.ListVCs()
for vc in vcList:
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vc["vcName"], Permission.User):
vc['admin'] = AuthorizationManager.HasAccess(userName, ResourceType.VC, vc["vcName"], Permission.Admin)
ret.append(vc)
# web portal (client) can filter out Default VC
return ret
def GetJobList(userName, vcName, jobOwner, num=None):
try:
dataHandler = DataHandler()
jobs = []
hasAccessOnAllJobs = False
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName,
Permission.Collaborator):
hasAccessOnAllJobs = True
if jobOwner != "all" or not hasAccessOnAllJobs:
jobs = jobs + GetUserPendingJobs(userName, vcName)
jobs = jobs + dataHandler.GetJobList(
userName, vcName, num,
"running,queued,scheduling,unapproved,pausing,paused",
("<>", "and"))
else:
jobs = GetUserPendingJobs(jobOwner, vcName)
for job in jobs:
job.pop('jobMeta', None)
dataHandler.Close()
return jobs
except Exception as e:
logger.error('Exception: %s', str(e))
logger.warn("Fail to get job list for user %s, return empty list",
userName)
return []
def GetJobDetail(userName, jobId):
job = None
dataHandler = DataHandler()
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Collaborator):
job = jobs[0]
job["log"] = ""
#jobParams = json.loads(base64.b64decode(job["jobMeta"]))
#jobPath,workPath,dataPath = GetStoragePath(jobParams["jobPath"],jobParams["workPath"],jobParams["dataPath"])
#localJobPath = os.path.join(config["storage-mount-path"],jobPath)
#logPath = os.path.join(localJobPath,"joblog.txt")
#print logPath
#if os.path.isfile(logPath):
# with open(logPath, 'r') as f:
# log = f.read()
# job["log"] = log
# f.close()
if "jobDescription" in job:
job.pop("jobDescription",None)
try:
log = dataHandler.GetJobTextField(jobId,"jobLog")
try:
if isBase64(log):
log = base64.b64decode(log)
except Exception:
pass
if log is not None:
job["log"] = log
except:
job["log"] = "fail-to-get-logs"
dataHandler.Close()
return job
def GetJobLog(userName, jobId):
dataHandler = DataHandler()
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(
userName, ResourceType.VC, jobs[0]["vcName"],
Permission.Collaborator):
try:
log = dataHandler.GetJobTextField(jobId, "jobLog")
try:
if isBase64(log):
log = base64.b64decode(log)
except Exception:
pass
if log is not None:
return {
"log": log,
"cursor": None,
}
except:
pass
return {
"log": {},
"cursor": None,
}
def ListStorages(userName, vcName):
ret = []
dataHandler = DataHandler()
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.User):
ret = dataHandler.ListStorages(vcName)
dataHandler.Close()
return ret
def GetCommands(userName, jobId):
commands = []
dataHandler = DataHandler()
jobs = dataHandler.GetJob(jobId=jobId)
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Collaborator):
commands = dataHandler.GetCommands(jobId=jobId)
dataHandler.Close()
return commands
def ResumeJob(userName, jobId):
dataHandler = DataHandler()
ret = False
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1 and jobs[0]["jobStatus"] == "paused":
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Collaborator):
ret = dataHandler.UpdateJobTextField(jobId, "jobStatus", "unapproved")
dataHandler.Close()
return ret
def PauseJob(userName, jobId):
dataHandler = DataHandler()
ret = False
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Admin):
ret = dataHandler.UpdateJobTextField(jobId,"jobStatus","pausing")
dataHandler.Close()
return ret
def AddCommand(userName, jobId,command):
dataHandler = DataHandler()
ret = False
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
if jobs[0]["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Collaborator):
ret = dataHandler.AddCommand(jobId,command)
dataHandler.Close()
return ret
def AddStorage(userName, vcName, url, storageType, metadata, defaultMountPath):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.IsClusterAdmin(userName):
ret = dataHandler.AddStorage(vcName, url, storageType, metadata, defaultMountPath)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def UpdateVC(userName, vcName, quota, metadata):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.IsClusterAdmin(userName):
ret = dataHandler.UpdateVC(vcName, quota, metadata)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def DeleteStorage(userName, vcName, url):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.Admin):
ret = dataHandler.DeleteStorage(vcName, url)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def UpdateStorage(userName, vcName, url, storageType, metadata, defaultMountPath):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.Admin):
ret = dataHandler.UpdateStorage(vcName, url, storageType, metadata, defaultMountPath)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def DeleteVC(userName, vcName):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.IsClusterAdmin(userName):
ret = dataHandler.DeleteVC(vcName)
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def ApproveJob(userName, jobId):
dataHandler = DataHandler()
ret = False
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
if AuthorizationManager.HasAccess(userName, ResourceType.VC, jobs[0]["vcName"], Permission.Admin):
ret = dataHandler.UpdateJobTextField(jobId,"jobStatus","queued")
dataHandler.Close()
InvalidateJobListCache(jobs[0]["vcName"])
return ret
def ApproveJob(userName, jobId):
dataHandler = DataHandler()
ret = False
job = dataHandler.GetJobTextFields(jobId, ["vcName", "jobStatus"])
if job is not None and job["jobStatus"] == "unapproved":
if AuthorizationManager.HasAccess(userName, ResourceType.VC,
job["vcName"], Permission.Admin):
ret = dataHandler.UpdateJobTextField(jobId, "jobStatus", "queued")
dataHandler.Close()
return ret
def GetCommands(userName, jobId):
commands = []
dataHandler = DataHandler()
job = dataHandler.GetJobTextFields(jobId, ["userName", "vcName"])
if job is not None:
if job["userName"] == userName or AuthorizationManager.HasAccess(
userName, ResourceType.VC, job["vcName"],
Permission.Collaborator):
commands = dataHandler.GetCommands(jobId=jobId)
dataHandler.Close()
return commands
def AddCommand(userName, jobId, command):
dataHandler = DataHandler()
ret = False
job = dataHandler.GetJobTextFields(jobId, ["userName", "vcName"])
if job is not None:
if job["userName"] == userName or AuthorizationManager.HasAccess(
userName, ResourceType.VC, job["vcName"],
Permission.Collaborator):
ret = dataHandler.AddCommand(jobId, command)
dataHandler.Close()
return ret
def get(self):
parser.add_argument('userName')
args = parser.parse_args()
username = args["userName"]
ret = {}
ret["result"] = AuthorizationManager.GetAcl(username)
resp = jsonify(ret)
resp.headers["Access-Control-Allow-Origin"] = "*"
resp.headers["dataType"] = "json"
return resp
def ListVCs(userName):
ret = []
dataHandler = DataHandler()
vcList = dataHandler.ListVCs()
for vc in vcList:
if AuthorizationManager.HasAccess(userName, ResourceType.VC, vc["vcName"], Permission.User):
# todo : get other info (resource consumption, quota etc.) about VC?
ret.append(vc)
# web portal (client) can filter out Default VC
dataHandler.Close()
return ret
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('vcName', location="args")
parser.add_argument('userName', location="args")
parser.add_argument('database', location="args")
parser.add_argument('templateName', location="args")
args = parser.parse_args()
vcName = args["vcName"]
userName = args["userName"]
database = args["database"]
templateName = args["templateName"]
if database == 'master':
if AuthorizationManager.HasAccess(userName, ResourceType.Cluster,
"", Permission.Admin):
scope = 'master'
else:
return 'access denied', 403
elif database == 'vc':
if AuthorizationManager.HasAccess(userName, ResourceType.VC,
vcName, Permission.Admin):
scope = 'vc:' + vcName
else:
return 'access denied', 403
else:
scope = 'user:' + userName
template_json = request.json
if template_json is None:
return jsonify(result=False, message="Invalid JSON")
dataHandler = DataHandler()
ret = {}
ret["result"] = dataHandler.UpdateTemplate(templateName, scope,
json.dumps(template_json))
dataHandler.Close()
resp = jsonify(ret)
resp.headers["Access-Control-Allow-Origin"] = "*"
resp.headers["dataType"] = "json"
return resp
def PauseJob(userName, jobId):
dataHandler = DataHandler()
ret = False
job = dataHandler.GetJobTextFields(jobId,
["userName", "vcName", "jobStatus"])
if job is not None and job["jobStatus"] in [
"unapproved", "queued", "scheduling", "running"
]:
if job["userName"] == userName or AuthorizationManager.HasAccess(
userName, ResourceType.VC, job["vcName"], Permission.Admin):
ret = dataHandler.UpdateJobTextField(jobId, "jobStatus", "pausing")
dataHandler.Close()
return ret
def KillJob(userName, jobId):
ret = False
dataHandler = DataHandler()
job = dataHandler.GetJobTextFields(
jobId, ["userName", "vcName", "jobStatus", "isParent", "familyToken"])
if job is not None and job["jobStatus"] in pendingStatus.split(","):
if job["userName"] == userName or AuthorizationManager.HasAccess(
userName, ResourceType.VC, job["vcName"], Permission.Admin):
dataFields = {"jobStatus": "killing"}
conditionFields = {"jobId": jobId}
if job["isParent"] == 1:
conditionFields = {"familyToken": job["familyToken"]}
ret = dataHandler.UpdateJobTextFields(conditionFields, dataFields)
dataHandler.Close()
return ret
def KillJob(userName, jobId):
ret = False
dataHandler = DataHandler()
jobs = dataHandler.GetJob(jobId=jobId)
if len(jobs) == 1:
job = jobs[0]
if job["userName"] == userName or AuthorizationManager.HasAccess(userName, ResourceType.VC, job["vcName"], Permission.Admin):
if job["isParent"] == 1:
ret = True
for currJob in dataHandler.GetJob(familyToken=job["familyToken"]):
ret = ret and dataHandler.UpdateJobTextField(currJob["jobId"],"jobStatus","killing")
else:
ret = dataHandler.UpdateJobTextField(jobId,"jobStatus","killing")
dataHandler.Close()
return ret
def UpdateVC(userName, vcName, quota, metadata):
ret = None
dataHandler = DataHandler()
if AuthorizationManager.IsClusterAdmin(userName):
ret = dataHandler.UpdateVC(vcName, quota, metadata)
if ret:
cacheItem = {
"vcName": vcName,
"quota": quota,
"metadata": metadata
}
with vc_cache_lock:
vc_cache[vcName] = cacheItem
else:
ret = "Access Denied!"
dataHandler.Close()
return ret
def GetVC(userName, vcName):
ret = None
clusterStatus, dummy = DataManager.GetClusterStatus()
clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
clusterReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"])
user_status = {}
vcList = DataManager.ListVCs()
for vc in vcList:
if vc["vcName"] == vcName and AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.User):
vcTotalRes = ResourceInfo(json.loads(vc["quota"]))
vcConsumedRes = ResourceInfo()
jobs = DataManager.GetAllPendingJobs(vcName)
for job in jobs:
if job["jobStatus"] == "running":
username = job["userName"]
jobParam = json.loads(base64.b64decode(job["jobParams"]))
if "gpuType" in jobParam and not jobParam["preemptionAllowed"]:
vcConsumedRes.Add(ResourceInfo({jobParam["gpuType"] : GetJobTotalGpu(jobParam)}))
if username not in user_status:
user_status[username] = ResourceInfo()
user_status[username].Add(ResourceInfo({jobParam["gpuType"] : GetJobTotalGpu(jobParam)}))
vcReservedRes = clusterReservedRes.GetFraction(vcTotalRes, clusterTotalRes)
vcAvailableRes = ResourceInfo.Difference(ResourceInfo.Difference(vcTotalRes, vcConsumedRes), vcReservedRes)
vc["gpu_capacity"] = vcTotalRes.ToSerializable()
vc["gpu_used"] = vcConsumedRes.ToSerializable()
vc["gpu_unschedulable"] = vcReservedRes.ToSerializable()
vc["gpu_avaliable"] = vcAvailableRes.ToSerializable()
vc["AvaliableJobNum"] = len(jobs)
vc["node_status"] = clusterStatus["node_status"]
vc["user_status"] = []
for user_name, user_gpu in user_status.iteritems():
# TODO: job_manager.getAlias should be put in a util file
user_name = user_name.split("@")[0].strip()
vc["user_status"].append({"userName":user_name, "userGPU":user_gpu.ToSerializable()})
ret = vc
break
return ret
def GetJobDetailV2(userName, jobId):
job = {}
dataHandler = None
try:
dataHandler = DataHandler()
jobs = dataHandler.GetJobV2(jobId)
if len(jobs) == 1:
if jobs[0][
"userName"] == userName or AuthorizationManager.HasAccess(
userName, ResourceType.VC, jobs[0]["vcName"],
Permission.Collaborator):
job = jobs[0]
except Exception as e:
logger.error(
"get job detail v2 exception for user: %s, jobId: %s, exception: %s",
userName, jobId, str(e))
finally:
if dataHandler is not None:
dataHandler.Close()
return job
