def TakeJobActions(jobs):
dataHandler = DataHandler()
vcList = dataHandler.ListVCs()
clusterStatus, _ = dataHandler.GetClusterStatus()
dataHandler.Close()
cluster_gpu_capacity = clusterStatus["gpu_capacity"]
cluster_gpu_reserved = clusterStatus["gpu_reserved"]
globalTotalRes = ResourceInfo(cluster_gpu_capacity)
globalReservedRes = ResourceInfo(cluster_gpu_reserved)
vc_resources = {}
localResInfo = ResourceInfo()
globalResInfo = ResourceInfo.Difference(globalTotalRes, globalReservedRes)
priority_dict = get_priority_dict()
logging.info("Job priority dict: {}".format(priority_dict))
for vc in vcList:
vcTotalRes = ResourceInfo(json.loads(vc["quota"]))
clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
clusterReservedRes = ResourceInfo(clusterStatus["gpu_reserved"])
vcReservedRes = clusterReservedRes.GetFraction(vcTotalRes,
clusterTotalRes)
vc_resources[vc["vcName"]] = ResourceInfo.Difference(
vcTotalRes, vcReservedRes)
jobsInfo = []
for job in jobs:
if job["jobStatus"] in ["queued", "scheduling", "running"]:
singleJobInfo = {}
singleJobInfo["job"] = job
job_params = json.loads(base64.b64decode(job["jobParams"]))
singleJobInfo["preemptionAllowed"] = job_params[
"preemptionAllowed"]
singleJobInfo["jobId"] = job_params["jobId"]
jobGpuType = "any"
if "gpuType" in job_params:
jobGpuType = job_params["gpuType"]
singleJobInfo["globalResInfo"] = ResourceInfo(
{jobGpuType: GetJobTotalGpu(job_params)})
# Job lists will be sorted based on and in the order of below
# 1. non-preemptible precedes preemptible
# 2. running precedes scheduling, precedes queued
# 3. larger priority value precedes lower priority value
# 4. early job time precedes later job time
# Non-Preemptible jobs first
preemptible = 1 if singleJobInfo["preemptionAllowed"] else 0
# Job status
job_status = 0
if job["jobStatus"] == "scheduling":
job_status = 1
elif job["jobStatus"] == "queued":
job_status = 2
# Priority value
reverse_priority = get_job_priority(priority_dict,
singleJobInfo["jobId"])
priority = 999999 - reverse_priority
# Job time
job_time = str(job["jobTime"])
singleJobInfo["sortKey"] = "{}_{}_{:06d}_{}".format(
preemptible, job_status, priority, job_time)
singleJobInfo["allowed"] = False
jobsInfo.append(singleJobInfo)
jobsInfo.sort(key=lambda x: x["sortKey"])
logging.info("TakeJobActions : local resources : %s" % (vc_resources))
logging.info("TakeJobActions : global resources : %s" %
(globalResInfo.CategoryToCountMap))
for sji in jobsInfo:
logging.info("TakeJobActions : job : %s : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap,
sji["sortKey"]))
vc_name = sji["job"]["vcName"]
vc_resource = vc_resources[vc_name]
if (vc_resource.CanSatisfy(sji["globalResInfo"])):
vc_resource.Subtract(sji["globalResInfo"])
globalResInfo.Subtract(sji["globalResInfo"])
sji["allowed"] = True
logging.info(
"TakeJobActions : local assignment : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap))
for sji in jobsInfo:
if sji["preemptionAllowed"] and (sji["allowed"] is False):
if globalResInfo.CanSatisfy(sji["globalResInfo"]):
logging.info(
"TakeJobActions : job : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap))
# Strict FIFO policy not required for global (bonus) tokens since these jobs are anyway pre-emptible.
globalResInfo.Subtract(sji["globalResInfo"])
sji["allowed"] = True
logging.info(
"TakeJobActions : global assignment : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap))
logging.info("TakeJobActions : global resources : %s" %
(globalResInfo.CategoryToCountMap))
for sji in jobsInfo:
try:
if sji["job"]["jobStatus"] == "queued" and (sji["allowed"] is
True):
SubmitJob(sji["job"])
logging.info("TakeJobActions : submitting job : %s : %s" %
(sji["jobId"], sji["sortKey"]))
elif sji["preemptionAllowed"] and (
sji["job"]["jobStatus"] == "scheduling"
or sji["job"]["jobStatus"]
== "running") and (sji["allowed"] is False):
KillJob(sji["job"]["jobId"], "queued")
logging.info("TakeJobActions : pre-empting job : %s : %s" %
(sji["jobId"], sji["sortKey"]))
except Exception as e:
logging.error("Process job failed {}".format(sji["job"]),
exc_info=True)
logging.info("TakeJobActions : job desired actions taken")
def get_cluster_status():
cluster_status = {}
gpuStr = "nvidia.com/gpu"
try:
output = k8sUtils.kubectl_exec(" get nodes -o yaml")
nodeInfo = yaml.load(output)
nodes_status = {}
user_status = {}
if "items" in nodeInfo:
for node in nodeInfo["items"]:
node_status = {}
node_status["name"] = node["metadata"]["name"]
node_status["labels"] = node["metadata"]["labels"]
node_status["gpuType"] = ""
node_status["scheduled_service"] = []
for l, s in node_status["labels"].iteritems():
if s == "active" and l != "all" and l != "default":
node_status["scheduled_service"].append(l)
if l == "gpuType":
node_status["scheduled_service"].append(s)
node_status["gpuType"] = s
if (gpuStr in node["status"]["allocatable"]):
node_status["gpu_allocatable"] = ResourceInfo({
node_status["gpuType"]:
int(node["status"]["allocatable"][gpuStr])
}).ToSerializable()
else:
node_status["gpu_allocatable"] = ResourceInfo(
).ToSerializable()
if (gpuStr in node["status"]["capacity"]):
node_status["gpu_capacity"] = ResourceInfo({
node_status["gpuType"]:
int(node["status"]["capacity"][gpuStr])
}).ToSerializable()
else:
node_status["gpu_capacity"] = ResourceInfo(
).ToSerializable()
node_status["gpu_used"] = ResourceInfo().ToSerializable()
node_status["InternalIP"] = "unknown"
node_status["pods"] = []
if "annotations" in node["metadata"]:
if "node.alpha/DeviceInformation" in node["metadata"][
"annotations"]:
node_info = json.loads(
node["metadata"]["annotations"]
["node.alpha/DeviceInformation"])
if (int(node_info["capacity"]["alpha.gpu/numgpu"]) >
ResourceInfo(
node_status["gpu_capacity"]).TotalCount()):
node_status["gpu_capacity"] = ResourceInfo({
node_status["gpuType"]:
int(node_info["capacity"]["alpha.gpu/numgpu"])
}).ToSerializable()
if (int(node_info["allocatable"]["alpha.gpu/numgpu"]) >
ResourceInfo(node_status["gpu_allocatable"]
).TotalCount()):
node_status["gpu_allocatable"] = ResourceInfo({
node_status["gpuType"]:
int(node_info["allocatable"]
["alpha.gpu/numgpu"])
}).ToSerializable()
if "addresses" in node["status"]:
for addr in node["status"]["addresses"]:
if addr["type"] == "InternalIP":
node_status["InternalIP"] = addr["address"]
if "unschedulable" in node["spec"] and node["spec"][
"unschedulable"]:
node_status["unschedulable"] = True
else:
node_status["unschedulable"] = False
if "status" in node and "conditions" in node["status"]:
for condi in node["status"]["conditions"]:
if "type" in condi and condi[
"type"] == "Ready" and "status" in condi and condi[
"status"] == "Unknown":
node_status["unschedulable"] = True
nodes_status[node_status["name"]] = node_status
output = k8sUtils.kubectl_exec(" get pods -o yaml")
podsInfo = yaml.load(output)
if "items" in podsInfo:
for pod in podsInfo["items"]:
gpus = 0
username = None
if "metadata" in pod and "labels" in pod[
"metadata"] and "userName" in pod["metadata"]["labels"]:
username = pod["metadata"]["labels"]["userName"]
if "spec" in pod and "nodeName" in pod["spec"]:
node_name = pod["spec"]["nodeName"]
pod_name = pod["metadata"]["name"]
if username is not None:
pod_name += " : " + username
gpuUsage = get_job_gpu_usage(pod["metadata"]["name"])
if gpuUsage is not None:
pod_name += " (gpu usage:" + str(gpuUsage) + "%)"
if gpuUsage <= 25:
pod_name += "!!!!!!"
pod_info_cont = {}
pod_info_initcont = {}
if "annotations" in pod["metadata"]:
if "pod.alpha/DeviceInformation" in pod["metadata"][
"annotations"]:
pod_info = json.loads(
pod["metadata"]["annotations"]
["pod.alpha/DeviceInformation"])
if "runningcontainer" in pod_info:
pod_info_cont = pod_info["runningcontainer"]
if "initcontainer" in pod_info:
pod_info_initcont = pod_info["initcontainer"]
if "containers" in pod["spec"]:
for container in pod["spec"]["containers"]:
containerGPUs = 0
if "resources" in container and "requests" in container[
"resources"] and gpuStr in container[
"resources"]["requests"]:
containerGPUs = int(
container["resources"]["requests"][gpuStr])
if container["name"] in pod_info_cont:
if "requests" in pod_info_cont[container[
"name"]] and "alpha.gpu/numgpu" in pod_info_cont[
container["name"]]["requests"]:
containerGPUs = max(
int(pod_info_cont[container["name"]]
["requests"]["alpha.gpu/numgpu"]),
containerGPUs)
gpus += containerGPUs
pod_name += " (gpu #:" + str(containerGPUs) + ")"
if node_name in nodes_status:
nodes_status[node_name]["gpu_used"] = ResourceInfo(
nodes_status[node_name]["gpu_used"]).Add(
ResourceInfo(
{nodes_status[node_name]["gpuType"]:
gpus})).ToSerializable()
nodes_status[node_name]["pods"].append(pod_name)
if username is not None:
if username not in user_status:
user_status[username] = ResourceInfo(
{nodes_status[node_name]["gpuType"]: gpus})
else:
user_status[username].Add(
ResourceInfo({
nodes_status[node_name]["gpuType"]:
gpus
}))
gpu_avaliable = ResourceInfo()
gpu_reserved = ResourceInfo()
gpu_capacity = ResourceInfo()
gpu_unschedulable = ResourceInfo()
gpu_schedulable = ResourceInfo()
gpu_used = ResourceInfo()
for node_name, node_status in nodes_status.iteritems():
if node_status["unschedulable"]:
gpu_unschedulable.Add(ResourceInfo(
node_status["gpu_capacity"]))
gpu_reserved.Add(
ResourceInfo.Difference(
ResourceInfo(node_status["gpu_capacity"]),
ResourceInfo(node_status["gpu_used"])))
else:
gpu_avaliable.Add(
ResourceInfo.Difference(
ResourceInfo(node_status["gpu_allocatable"]),
ResourceInfo(node_status["gpu_used"])))
gpu_schedulable.Add(ResourceInfo(node_status["gpu_capacity"]))
gpu_unschedulable.Add(
ResourceInfo.Difference(
ResourceInfo(node_status["gpu_capacity"]),
ResourceInfo(node_status["gpu_allocatable"])))
gpu_reserved.Add(
ResourceInfo.Difference(
ResourceInfo(node_status["gpu_capacity"]),
ResourceInfo(node_status["gpu_allocatable"])))
gpu_used.Add(ResourceInfo(node_status["gpu_used"]))
gpu_capacity.Add(ResourceInfo(node_status["gpu_capacity"]))
cluster_status["user_status"] = []
for user_name, user_gpu in user_status.iteritems():
cluster_status["user_status"].append({
"userName":
user_name,
"userGPU":
user_gpu.ToSerializable()
})
cluster_status["gpu_avaliable"] = gpu_avaliable.ToSerializable()
cluster_status["gpu_capacity"] = gpu_capacity.ToSerializable()
cluster_status["gpu_unschedulable"] = gpu_unschedulable.ToSerializable(
)
cluster_status["gpu_used"] = gpu_used.ToSerializable()
cluster_status["gpu_reserved"] = gpu_reserved.ToSerializable()
cluster_status["node_status"] = [
node_status for node_name, node_status in nodes_status.iteritems()
]
except Exception as e:
logging.exception("get cluster status")
dataHandler = DataHandler()
cluster_status["AvaliableJobNum"] = dataHandler.GetActiveJobsCount()
if "cluster_status" in config and check_cluster_status_change(
config["cluster_status"], cluster_status):
logging.info("updating the cluster status...")
dataHandler.UpdateClusterStatus(cluster_status)
else:
logging.info(
"nothing changed in cluster, skipping the cluster status update..."
)
config["cluster_status"] = copy.deepcopy(cluster_status)
dataHandler.Close()
return cluster_status
def TakeJobActions(jobs):
dataHandler = DataHandler()
vcList = dataHandler.ListVCs()
clusterStatus, dummy = dataHandler.GetClusterStatus()
dataHandler.Close()
globalTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
globalReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"])
localResInfo = ResourceInfo()
globalResInfo = ResourceInfo.Difference(globalTotalRes, globalReservedRes)
for vc in vcList:
vcTotalRes = ResourceInfo(json.loads(vc["quota"]), vc["vcName"])
clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"],
vc["vcName"])
clusterReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"],
vc["vcName"])
vcReservedRes = clusterReservedRes.GetFraction(vcTotalRes,
clusterTotalRes)
localResInfo.Add(ResourceInfo.Difference(vcTotalRes, vcReservedRes))
jobsInfo = []
for job in jobs:
if job["jobStatus"] == "queued" or job[
"jobStatus"] == "scheduling" or job["jobStatus"] == "running":
singleJobInfo = {}
singleJobInfo["job"] = job
singleJobInfo["jobParams"] = json.loads(
base64.b64decode(job["jobParams"]))
jobGpuType = "any"
if "gpuType" in singleJobInfo["jobParams"]:
jobGpuType = singleJobInfo["jobParams"]["gpuType"]
singleJobInfo["localResInfo"] = ResourceInfo(
{jobGpuType: GetJobTotalGpu(singleJobInfo["jobParams"])},
job["vcName"])
singleJobInfo["globalResInfo"] = ResourceInfo(
{jobGpuType: GetJobTotalGpu(singleJobInfo["jobParams"])})
singleJobInfo["sortKey"] = str(job["jobTime"])
if singleJobInfo["jobParams"]["preemptionAllowed"]:
singleJobInfo["sortKey"] = "1_" + singleJobInfo["sortKey"]
else:
singleJobInfo["sortKey"] = "0_" + singleJobInfo["sortKey"]
singleJobInfo["allowed"] = False
jobsInfo.append(singleJobInfo)
jobsInfo.sort(key=JobInfoSorter)
logging.info("TakeJobActions : local resources : %s" %
(localResInfo.CategoryToCountMap))
logging.info("TakeJobActions : global resources : %s" %
(globalResInfo.CategoryToCountMap))
for sji in jobsInfo:
logging.info("TakeJobActions : job : %s : %s : %s" %
(sji["jobParams"]["jobName"],
sji["localResInfo"].CategoryToCountMap, sji["sortKey"]))
if sji["jobParams"]["preemptionAllowed"]:
localResInfo.UnblockResourceCategory(sji["localResInfo"])
if (localResInfo.CanSatisfy(sji["localResInfo"])):
localResInfo.Subtract(sji["localResInfo"])
globalResInfo.Subtract(sji["globalResInfo"])
sji["allowed"] = True
logging.info("TakeJobActions : local assignment : %s : %s" %
(sji["jobParams"]["jobName"],
sji["localResInfo"].CategoryToCountMap))
elif not sji["jobParams"]["preemptionAllowed"]:
localResInfo.BlockResourceCategory(
sji["localResInfo"]) #FIFO scheduling
#logging.info("TakeJobActions : local resources : %s" % (localResInfo.CategoryToCountMap))
#logging.info("TakeJobActions : global resources : %s" % (globalResInfo.CategoryToCountMap))
for sji in jobsInfo:
if (sji["jobParams"]["preemptionAllowed"] and sji["allowed"] == False):
if globalResInfo.CanSatisfy(sji["globalResInfo"]):
logging.info("TakeJobActions : job : %s : %s" %
(sji["jobParams"]["jobName"],
sji["globalResInfo"].CategoryToCountMap))
# Strict FIFO policy not required for global (bonus) tokens since these jobs are anyway pre-emptible.
globalResInfo.Subtract(sji["globalResInfo"])
sji["allowed"] = True
logging.info("TakeJobActions : global assignment : %s : %s" %
(sji["jobParams"]["jobName"],
sji["globalResInfo"].CategoryToCountMap))
logging.info("TakeJobActions : global resources : %s" %
(globalResInfo.CategoryToCountMap))
for sji in jobsInfo:
if sji["job"]["jobStatus"] == "queued" and sji["allowed"] == True:
SubmitJob(sji["job"])
logging.info("TakeJobActions : submitting job : %s : %s : %s" %
(sji["jobParams"]["jobName"],
sji["jobParams"]["jobId"], sji["sortKey"]))
elif sji["jobParams"]["preemptionAllowed"] and (
sji["job"]["jobStatus"] == "scheduling"
or sji["job"]["jobStatus"]
== "running") and sji["allowed"] == False:
KillJob(sji["job"], "queued")
logging.info("TakeJobActions : pre-empting job : %s : %s : %s" %
(sji["jobParams"]["jobName"],
sji["jobParams"]["jobId"], sji["sortKey"]))
logging.info("TakeJobActions : job desired actions taken")
def GetVC(userName, vcName):
ret = None
clusterStatus, dummy = DataManager.GetClusterStatus()
clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
clusterReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"])
user_status = {}
vcList = DataManager.ListVCs()
for vc in vcList:
if vc["vcName"] == vcName and AuthorizationManager.HasAccess(userName, ResourceType.VC, vcName, Permission.User):
vcTotalRes = ResourceInfo(json.loads(vc["quota"]))
vcConsumedRes = ResourceInfo()
jobs = DataManager.GetAllPendingJobs(vcName)
for job in jobs:
if job["jobStatus"] == "running":
username = job["userName"]
jobParam = json.loads(base64.b64decode(job["jobParams"]))
if "gpuType" in jobParam and not jobParam["preemptionAllowed"]:
vcConsumedRes.Add(ResourceInfo({jobParam["gpuType"] : GetJobTotalGpu(jobParam)}))
if username not in user_status:
user_status[username] = ResourceInfo()
user_status[username].Add(ResourceInfo({jobParam["gpuType"] : GetJobTotalGpu(jobParam)}))
vcReservedRes = clusterReservedRes.GetFraction(vcTotalRes, clusterTotalRes)
vcAvailableRes = ResourceInfo.Difference(ResourceInfo.Difference(vcTotalRes, vcConsumedRes), vcReservedRes)
vc["gpu_capacity"] = vcTotalRes.ToSerializable()
vc["gpu_used"] = vcConsumedRes.ToSerializable()
vc["gpu_unschedulable"] = vcReservedRes.ToSerializable()
vc["gpu_avaliable"] = vcAvailableRes.ToSerializable()
vc["AvaliableJobNum"] = len(jobs)
vc["node_status"] = clusterStatus["node_status"]
vc["user_status"] = []
for user_name, user_gpu in user_status.iteritems():
# TODO: job_manager.getAlias should be put in a util file
user_name = user_name.split("@")[0].strip()
vc["user_status"].append({"userName":user_name, "userGPU":user_gpu.ToSerializable()})
ret = vc
break
return ret
def GetVC(userName, vcName):
ret = None
data_handler = DataHandler()
cluster_status, _ = data_handler.GetClusterStatus()
cluster_total = cluster_status["gpu_capacity"]
cluster_available = cluster_status["gpu_avaliable"]
cluster_reserved = cluster_status["gpu_reserved"]
user_status = collections.defaultdict(lambda: ResourceInfo())
user_status_preemptable = collections.defaultdict(lambda: ResourceInfo())
vc_list = getClusterVCs()
vc_info = {}
vc_usage = collections.defaultdict(
lambda: collections.defaultdict(lambda: 0))
vc_preemptable_usage = collections.defaultdict(
lambda: collections.defaultdict(lambda: 0))
for vc in vc_list:
vc_info[vc["vcName"]] = json.loads(vc["quota"])
active_job_list = data_handler.GetActiveJobList()
for job in active_job_list:
jobParam = json.loads(base64.b64decode(job["jobParams"]))
if "gpuType" in jobParam:
if not jobParam["preemptionAllowed"]:
vc_usage[job["vcName"]][jobParam["gpuType"]] += GetJobTotalGpu(
jobParam)
else:
vc_preemptable_usage[job["vcName"]][
jobParam["gpuType"]] += GetJobTotalGpu(jobParam)
result = quota.calculate_vc_gpu_counts(cluster_total, cluster_available,
cluster_reserved, vc_info, vc_usage)
vc_total, vc_used, vc_available, vc_unschedulable = result
for vc in vc_list:
if vc["vcName"] == vcName and AuthorizationManager.HasAccess(
userName, ResourceType.VC, vcName, Permission.User):
num_active_jobs = 0
for job in active_job_list:
if job["vcName"] == vcName and job["jobStatus"] == "running":
num_active_jobs += 1
username = job["userName"]
jobParam = json.loads(base64.b64decode(job["jobParams"]))
if "gpuType" in jobParam:
if not jobParam["preemptionAllowed"]:
if username not in user_status:
user_status[username] = ResourceInfo()
user_status[username].Add(
ResourceInfo({
jobParam["gpuType"]:
GetJobTotalGpu(jobParam)
}))
else:
if username not in user_status_preemptable:
user_status_preemptable[
username] = ResourceInfo()
user_status_preemptable[username].Add(
ResourceInfo({
jobParam["gpuType"]:
GetJobTotalGpu(jobParam)
}))
vc["gpu_capacity"] = vc_total[vcName]
vc["gpu_used"] = vc_used[vcName]
vc["gpu_preemptable_used"] = vc_preemptable_usage[vcName]
vc["gpu_unschedulable"] = vc_unschedulable[vcName]
vc["gpu_avaliable"] = vc_available[vcName]
vc["AvaliableJobNum"] = num_active_jobs
vc["node_status"] = cluster_status["node_status"]
vc["user_status"] = []
for user_name, user_gpu in user_status.iteritems():
# TODO: job_manager.getAlias should be put in a util file
user_name = user_name.split("@")[0].strip()
vc["user_status"].append({
"userName": user_name,
"userGPU": user_gpu.ToSerializable()
})
vc["user_status_preemptable"] = []
for user_name, user_gpu in user_status_preemptable.iteritems():
user_name = user_name.split("@")[0].strip()
vc["user_status_preemptable"].append({
"userName":
user_name,
"userGPU":
user_gpu.ToSerializable()
})
try:
gpu_idle_url = config["gpu_reporter"] + '/gpu_idle'
gpu_idle_params = {"vc": vcName}
gpu_idle_response = requests.get(gpu_idle_url,
params=gpu_idle_params)
gpu_idle_json = gpu_idle_response.json()
vc["gpu_idle"] = gpu_idle_json
except Exception:
logger.exception("Failed to fetch gpu_idle from gpu-exporter")
ret = vc
break
return ret
def TakeJobActions(data_handler, redis_conn, launcher, jobs):
vc_list = data_handler.ListVCs()
cluster_status, _ = data_handler.GetClusterStatus()
cluster_total = cluster_status["gpu_capacity"]
cluster_available = cluster_status["gpu_avaliable"]
cluster_reserved = cluster_status["gpu_reserved"]
vc_info = {}
vc_usage = collections.defaultdict(
lambda: collections.defaultdict(lambda: 0))
for vc in vc_list:
vc_info[vc["vcName"]] = json.loads(vc["quota"])
active_job_list = data_handler.GetActiveJobList()
for job in active_job_list:
jobParam = json.loads(base64.b64decode(job["jobParams"]))
if "gpuType" in jobParam:
vc_usage[job["vcName"]][jobParam["gpuType"]] += GetJobTotalGpu(
jobParam)
result = quota.calculate_vc_gpu_counts(cluster_total, cluster_available,
cluster_reserved, vc_info, vc_usage)
vc_total, vc_used, vc_available, vc_unschedulable = result
cluster_gpu_capacity = cluster_status["gpu_capacity"]
cluster_gpu_unschedulable = cluster_status["gpu_unschedulable"]
global_total = ResourceInfo(cluster_gpu_capacity)
global_unschedulable = ResourceInfo(cluster_gpu_unschedulable)
vc_resources = {}
globalResInfo = ResourceInfo.Difference(global_total, global_unschedulable)
priority_dict = get_priority_dict()
logging.info("Job priority dict: {}".format(priority_dict))
for vc in vc_list:
vc_name = vc["vcName"]
vc_schedulable = {}
for gpu_type, total in vc_total[vc_name].items():
vc_schedulable[
gpu_type] = total - vc_unschedulable[vc_name][gpu_type]
vc_resources[vc_name] = ResourceInfo(vc_schedulable)
jobsInfo = []
for job in jobs:
if job["jobStatus"] in ["queued", "scheduling", "running"]:
singleJobInfo = {}
singleJobInfo["job"] = job
job_params = json.loads(base64.b64decode(job["jobParams"]))
singleJobInfo["preemptionAllowed"] = job_params[
"preemptionAllowed"]
singleJobInfo["jobId"] = job_params["jobId"]
jobGpuType = "any"
if "gpuType" in job_params:
jobGpuType = job_params["gpuType"]
singleJobInfo["globalResInfo"] = ResourceInfo(
{jobGpuType: GetJobTotalGpu(job_params)})
# Job lists will be sorted based on and in the order of below
# 1. non-preemptible precedes preemptible
# 2. running precedes scheduling, precedes queued
# 3. larger priority value precedes lower priority value
# 4. early job time precedes later job time
# Non-Preemptible jobs first
preemptible = 1 if singleJobInfo["preemptionAllowed"] else 0
# Job status
job_status = 0
if job["jobStatus"] == "scheduling":
job_status = 1
elif job["jobStatus"] == "queued":
job_status = 2
# Priority value
reverse_priority = get_job_priority(priority_dict,
singleJobInfo["jobId"])
priority = 999999 - reverse_priority
# Job time
job_time = str(job["jobTime"])
singleJobInfo["sortKey"] = "{}_{}_{:06d}_{}".format(
preemptible, job_status, priority, job_time)
singleJobInfo["allowed"] = False
jobsInfo.append(singleJobInfo)
jobsInfo.sort(key=lambda x: x["sortKey"])
logging.info("TakeJobActions : local resources : %s" % (vc_resources))
logging.info("TakeJobActions : global resources : %s" %
(globalResInfo.CategoryToCountMap))
for sji in jobsInfo:
logging.info("TakeJobActions : job : %s : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap,
sji["sortKey"]))
vc_name = sji["job"]["vcName"]
vc_resource = vc_resources[vc_name]
if (not sji["preemptionAllowed"]) and (vc_resource.CanSatisfy(
sji["globalResInfo"])):
vc_resource.Subtract(sji["globalResInfo"])
globalResInfo.Subtract(sji["globalResInfo"])
sji["allowed"] = True
logging.info(
"TakeJobActions : local assignment : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap))
for sji in jobsInfo:
if sji["preemptionAllowed"] and (sji["allowed"] is False):
if globalResInfo.CanSatisfy(sji["globalResInfo"]):
logging.info(
"TakeJobActions : job : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap))
# Strict FIFO policy not required for global (bonus) tokens since these jobs are anyway pre-emptible.
globalResInfo.Subtract(sji["globalResInfo"])
sji["allowed"] = True
logging.info(
"TakeJobActions : global assignment : %s : %s" %
(sji["jobId"], sji["globalResInfo"].CategoryToCountMap))
logging.info("TakeJobActions : global resources : %s" %
(globalResInfo.CategoryToCountMap))
for sji in jobsInfo:
try:
if sji["job"]["jobStatus"] == "queued" and (sji["allowed"] is
True):
launcher.submit_job(sji["job"])
update_job_state_latency(redis_conn, sji["jobId"],
"scheduling")
logging.info("TakeJobActions : submitting job : %s : %s" %
(sji["jobId"], sji["sortKey"]))
elif sji["preemptionAllowed"] and (
sji["job"]["jobStatus"] == "scheduling"
or sji["job"]["jobStatus"]
== "running") and (sji["allowed"] is False):
launcher.kill_job(sji["job"]["jobId"], "queued")
logging.info("TakeJobActions : pre-empting job : %s : %s" %
(sji["jobId"], sji["sortKey"]))
except Exception as e:
logging.error("Process job failed {}".format(sji["job"]),
exc_info=True)
logging.info("TakeJobActions : job desired actions taken")