def to_dict(self):
d = {
'job_id': self.job_id,
'executable': self.executable,
'files': self.files,
'status': self.status,
'walltime': walltime.strftime(self.wall_time),
'deadline': self.deadline,
'flags': self.flags,
'budget': self.budget,
'budget_per_node_hour': self.budget_per_node_hour,
'job_type': self.job_type,
'created_ts': self.created_ts,
'ready_ts': self.ready_ts,
'running_ts': self.running_ts,
'finished_ts': self.finished_ts,
'name': self.name,
'kill_msg': self.kill_msg,
'work_units': [],
}
for work_unit in self.work_units:
d['work_units'].append(work_unit.to_dict())
return d
def write_queue_to_log(self): job_queue = defaultdict(list) # No work units, dont print. if len(self.grid.get_queued()) == 0: return # Build Queue by Job ID for unit in self.grid.get_queued(): job_queue[unit.job.job_id].append(unit) # Print out relevant information for each job queue_string = "Current jobs waiting for allocation:\n" for job_id, units in job_queue.items(): # Print information about the job created_ts = time.asctime(time.localtime(units[0].job.created_ts)) queue_string += "Job: %s.\n" % (job_id) queue_string += "Type: %s.\n" % (units[0].job.job_type) queue_string += "Creation Time: %s.\n" % (created_ts) queue_string += "Wall Time: %s.\n" % walltime.strftime(units[0].job.wall_time) queue_string += "Deadline: %s.\n" % time.asctime(time.localtime(units[0].job.deadline)) queue_string += "Total Budget: $%0.2f.\n" % (units[0].job.budget/100) queue_string += "Budget per node hour: $%0.2f.\n" % (units[0].job.budget_per_node_hour/100) # Print out a job's currently queued work units queue_string += "Work Units: [" for unit in units: queue_string += "%s, " % (unit.work_unit_id) queue_string = "%s]\n\n" % (queue_string[0:-2]) # -2 drops the last ", " # Write out to log self.write_to_log(queue_string)
def allocate_work_unit(self, node, work_unit):
try:
url = '%s/task' % (self.grid.get_node_url(node))
request = JSONHTTPRequest( 'POST', url, {
'work_unit_id': work_unit.work_unit_id,
'job_id': work_unit.job.job_id,
'executable': work_unit.job.executable,
'filename': work_unit.filename,
'flags': work_unit.job.flags,
'wall_time': walltime.strftime(work_unit.job.wall_time),
'deadline': work_unit.job.deadline,
}, self.grid.auth_header)
except (HTTPException, URLError) as e:
raise NodeUnavailableException("The node at %s is unavailable." % self.grid.get_node_url(node))
d = request.response
work_unit.running(node['node_id'], d['task_id'])
node['work_units'].append(work_unit)
def to_dict(self):
d = {
'work_unit_id': self.work_unit_id,
'job_id': self.job.job_id,
'executable': self.job.executable,
'flags': self.job.flags,
'filename': self.filename,
'wall_time': walltime.strftime(self.job.wall_time),
'job_type': self.job.job_type,
'kill_msg': self.kill_msg,
'node_id': self.node_id,
'status': self.status,
'cost': self.cost,
'created_ts': self.created_ts,
'finished_ts': self.finished_ts,
}
return d
def add_job(self, flags, wall_time, deadline, budget, job_type, name):
# Need to check job_type is a valid queue
if job_type is None:
job_type = "DEFAULT"
elif job_type not in self.node_queue.keys():
raise InvalidJobTypeException(
"Invalid Job Type specified: %s. Valid job types are: %s." %
(job_type, ", ".join(self.node_queue.keys())))
# Check for Valid budget
try:
budget = int(budget)
except (TypeError, ValueError):
raise InvalidJobBudgetException(
"Invalid Budget specified: %s. Format: amount in cents as a whole number."
% budget)
if budget < 0:
raise InvalidJobBudgetException(
"Invalid Budget specified: %s. Budget must be greater than 0" %
budget)
# Check that wall_time is valid:
try:
wall_stripped = walltime.strptime(wall_time)
except WallTimeFormatException:
raise InvalidWallTimeFormatException(
"Invalid Wall Time specified: %s. Format: DD:HH:MM:SS." %
wall_time)
# Check that deadline format is valid
try:
deadline_since_epoch = time.mktime(
time.strptime(deadline, "%Y-%m-%d %H:%M:%S"))
except ValueError:
raise InvalidJobDeadlineFormatException(
"Invalid Deadline specified: %s. Format: YYYY-MM-DD HH:MM:SS" %
deadline)
# Check that deadline is valid
if deadline_since_epoch <= int(time.time()):
raise InvalidJobDeadlineException(
"Invalid Deadline specified: %s. Deadline specified is in the past."
% deadline)
# Check that deadline is reasonable
if (deadline_since_epoch - walltime.wall_secs(wall_stripped)) < int(
time.time()):
raise InvalidJobDeadlineException(
"Error: Current time plus wall time is later than the specified deadline. Please adjust either and resubmit."
)
# Check that wall time is within acceptable range for job queue placement
if self.node_queue[job_type][1] != None and walltime.wall_secs(
wall_stripped) > walltime.wall_secs(
self.node_queue[job_type][1]):
raise InvalidJobTypeException(
"Invalid Job Type specified: %s. Wall time %s is too large. Wall time must be shorter than %s for job type %s."
% (job_type, walltime.strftime(wall_stripped),
self.node_queue[job_type][1], job_type))
#
# All tests passed, add to grid.
#
job = Job(job_id=self.next_job_id,
flags=flags,
wall_time=wall_stripped,
deadline=deadline_since_epoch,
budget=budget,
job_type=job_type,
name=name)
self.jobs[self.next_job_id] = job
self.next_job_id += 1
return job
def add_job(self, flags, wall_time, deadline, budget, job_type, name):
# Need to check job_type is a valid queue
if job_type is None:
job_type = "DEFAULT"
elif job_type not in self.node_queue.keys():
raise InvalidJobTypeException(
"Invalid Job Type specified: %s. Valid job types are: %s." % (job_type, ", ".join(self.node_queue.keys()))
)
# Check for Valid budget
try:
budget = int(budget)
except (TypeError, ValueError):
raise InvalidJobBudgetException("Invalid Budget specified: %s. Format: amount in cents as a whole number." % budget)
if budget < 0:
raise InvalidJobBudgetException("Invalid Budget specified: %s. Budget must be greater than 0" % budget)
# Check that wall_time is valid:
try:
wall_stripped = walltime.strptime(wall_time)
except WallTimeFormatException:
raise InvalidWallTimeFormatException("Invalid Wall Time specified: %s. Format: DD:HH:MM:SS." % wall_time)
# Check that deadline format is valid
try:
deadline_since_epoch = time.mktime(time.strptime(deadline, "%Y-%m-%d %H:%M:%S"))
except ValueError:
raise InvalidJobDeadlineFormatException("Invalid Deadline specified: %s. Format: YYYY-MM-DD HH:MM:SS" % deadline)
# Check that deadline is valid
if deadline_since_epoch <= int(time.time()):
raise InvalidJobDeadlineException("Invalid Deadline specified: %s. Deadline specified is in the past." % deadline)
# Check that deadline is reasonable
if (deadline_since_epoch - walltime.wall_secs(wall_stripped)) < int(time.time()):
raise InvalidJobDeadlineException(
"Error: Current time plus wall time is later than the specified deadline. Please adjust either and resubmit."
)
# Check that wall time is within acceptable range for job queue placement
if self.node_queue[job_type][1] != None and walltime.wall_secs(wall_stripped) > walltime.wall_secs(self.node_queue[job_type][1]):
raise InvalidJobTypeException(
"Invalid Job Type specified: %s. Wall time %s is too large. Wall time must be shorter than %s for job type %s."
% (job_type, walltime.strftime(wall_stripped), self.node_queue[job_type][1], job_type)
)
#
# All tests passed, add to grid.
#
job = Job(
job_id = self.next_job_id,
flags = flags,
wall_time = wall_stripped,
deadline = deadline_since_epoch,
budget = budget,
job_type = job_type,
name = name
)
self.jobs[ self.next_job_id ] = job
self.next_job_id += 1
return job
