def buildElement(element, items, itemName):
""" Create an element for output.
"""
def assertNonnegative(i,name):
if i < 0:
raise RuntimeError("Negative value %s reported for %s" %(i,name) )
else:
return float(i)
itemTimes = []
itemClocks = []
itemMemory = []
for item in items:
itemTimes.append(assertNonnegative(float(item["time"]), "time"))
itemClocks.append(assertNonnegative(float(item["clock"]), "clock"))
itemMemory.append(assertNonnegative(float(item["memory"]), "memory"))
assert len(itemClocks) == len(itemTimes) == len(itemMemory)
itemWaits=[]
for index in range(0,len(itemTimes)):
itemWaits.append(itemTimes[index] - itemClocks[index])
itemWaits.sort()
itemTimes.sort()
itemClocks.sort()
itemMemory.sort()
if len(itemTimes) == 0:
itemTimes.append(0)
itemClocks.append(0)
itemWaits.append(0)
itemMemory.append(0)
element[itemName]=Expando(
total_number=float(len(items)),
total_time=float(sum(itemTimes)),
median_time=float(itemTimes[old_div(len(itemTimes),2)]),
average_time=float(old_div(sum(itemTimes),len(itemTimes))),
min_time=float(min(itemTimes)),
max_time=float(max(itemTimes)),
total_clock=float(sum(itemClocks)),
median_clock=float(itemClocks[old_div(len(itemClocks),2)]),
average_clock=float(old_div(sum(itemClocks),len(itemClocks))),
min_clock=float(min(itemClocks)),
max_clock=float(max(itemClocks)),
total_wait=float(sum(itemWaits)),
median_wait=float(itemWaits[old_div(len(itemWaits),2)]),
average_wait=float(old_div(sum(itemWaits),len(itemWaits))),
min_wait=float(min(itemWaits)),
max_wait=float(max(itemWaits)),
total_memory=float(sum(itemMemory)),
median_memory=float(itemMemory[old_div(len(itemMemory),2)]),
average_memory=float(old_div(sum(itemMemory),len(itemMemory))),
min_memory=float(min(itemMemory)),
max_memory=float(max(itemMemory)),
name=itemName
)
return element[itemName]
def processData(config, stats):
"""
Collate the stats and report
"""
if 'total_time' not in stats or 'total_clock' not in stats:
# toil job not finished yet
stats.total_time = [0.0]
stats.total_clock = [0.0]
stats.total_time = sum([float(number) for number in stats.total_time])
stats.total_clock = sum([float(number) for number in stats.total_clock])
collatedStatsTag = Expando(total_run_time=stats.total_time,
total_clock=stats.total_clock,
batch_system=config.batchSystem,
default_memory=str(config.defaultMemory),
default_cores=str(config.defaultCores),
max_cores=str(config.maxCores)
)
# Add worker info
worker = [_f for _f in getattr(stats, 'workers', []) if _f]
jobs = [_f for _f in getattr(stats, 'jobs', []) if _f]
jobs = [item for sublist in jobs for item in sublist]
def fn4(job):
try:
return list(jobs)
except TypeError:
return []
buildElement(collatedStatsTag, worker, "worker")
createSummary(buildElement(collatedStatsTag, jobs, "jobs"),
getattr(stats, 'workers', []), "worker", fn4)
# Get info for each job
jobNames = set()
for job in jobs:
jobNames.add(job.class_name)
jobTypesTag = Expando()
collatedStatsTag.job_types = jobTypesTag
for jobName in jobNames:
jobTypes = [ job for job in jobs if job.class_name == jobName ]
buildElement(jobTypesTag, jobTypes, jobName)
collatedStatsTag.name = "collatedStatsTag"
return collatedStatsTag
def processData(config, stats):
##########################################
# Collate the stats and report
##########################################
if stats.get("total_time",
None) is None: # Hack to allow unfinished toils.
stats.total_time = {"total_time": "0.0", "total_clock": "0.0"}
else:
stats.total_time = sum([float(number) for number in stats.total_time])
stats.total_clock = sum(
[float(number) for number in stats.total_clock])
collatedStatsTag = Expando(total_run_time=stats.total_time,
total_clock=stats.total_clock,
batch_system=config.batchSystem,
default_memory=str(config.defaultMemory),
default_cores=str(config.defaultCores),
max_cores=str(config.maxCores))
# Add worker info
worker = [_f for _f in stats.workers if _f]
jobs = [_f for _f in stats.jobs if _f]
jobs = [item for sublist in jobs for item in sublist]
def fn4(job):
try:
return list(jobs)
except TypeError:
return []
buildElement(collatedStatsTag, worker, "worker")
createSummary(buildElement(collatedStatsTag, jobs, "jobs"), stats.workers,
"worker", fn4)
# Get info for each job
jobNames = set()
for job in jobs:
jobNames.add(job.class_name)
jobTypesTag = Expando()
collatedStatsTag.job_types = jobTypesTag
for jobName in jobNames:
jobTypes = [job for job in jobs if job.class_name == jobName]
buildElement(jobTypesTag, jobTypes, jobName)
collatedStatsTag.name = "collatedStatsTag"
return collatedStatsTag
def _sendFrameworkMessage(self, driver):
message = None
while True:
# The psutil documentation recommends that we ignore the value returned by the first
# invocation of cpu_percent(). However, we do want to send a sign of life early after
# starting (e.g. to unblock the provisioner waiting for an instance to come up) so
# we call it once and discard the value.
if message is None:
message = Expando(address=self.address)
psutil.cpu_percent()
else:
message.nodeInfo = dict(coresUsed=float(psutil.cpu_percent()) * .01,
memoryUsed=float(psutil.virtual_memory().percent) * .01,
coresTotal=psutil.cpu_count(),
memoryTotal=psutil.virtual_memory().total,
workers=len(self.runningTasks))
log.debug("Send framework message: %s", message)
driver.sendFrameworkMessage(encode_data(repr(message)))
# Prevent workers launched together from repeatedly hitting the leader at the same time
time.sleep(random.randint(45, 75))
def getStats(jobStore):
""" Collect and return the stats and config data.
"""
def aggregateStats(fileHandle,aggregateObject):
try:
stats = json.load(fileHandle, object_hook=Expando)
for key in list(stats.keys()):
if key in aggregateObject:
aggregateObject[key].append(stats[key])
else:
aggregateObject[key]=[stats[key]]
except ValueError:
logger.critical("File %s contains corrupted json. Skipping file." % fileHandle)
pass # The file is corrupted.
aggregateObject = Expando()
callBack = partial(aggregateStats, aggregateObject=aggregateObject)
jobStore.readStatsAndLogging(callBack, readAll=True)
return aggregateObject
def buildElement(element: Expando, items: List[Job], itemName: str) -> Expando:
""" Create an element for output.
"""
def assertNonnegative(i: float, name: str) -> float:
if i < 0:
raise RuntimeError("Negative value %s reported for %s" % (i, name))
else:
return float(i)
itemTimes = []
itemClocks = []
itemMemory = []
for item in items:
# If something lacks an entry, assume it used none of that thing.
# This avoids crashing when jobs e.g. aren't done.
itemTimes.append(assertNonnegative(float(item.get("time", 0)), "time"))
itemClocks.append(
assertNonnegative(float(item.get("clock", 0)), "clock"))
itemMemory.append(
assertNonnegative(float(item.get("memory", 0)), "memory"))
assert len(itemClocks) == len(itemTimes) == len(itemMemory)
itemWaits = []
for index in range(0, len(itemTimes)):
itemWaits.append(itemTimes[index] - itemClocks[index])
itemWaits.sort()
itemTimes.sort()
itemClocks.sort()
itemMemory.sort()
if len(itemTimes) == 0:
itemTimes.append(0)
itemClocks.append(0)
itemWaits.append(0)
itemMemory.append(0)
element[itemName] = Expando(
total_number=float(len(items)),
total_time=float(sum(itemTimes)),
median_time=float(itemTimes[len(itemTimes) // 2]),
average_time=float(sum(itemTimes) / len(itemTimes)),
min_time=float(min(itemTimes)),
max_time=float(max(itemTimes)),
total_clock=float(sum(itemClocks)),
median_clock=float(itemClocks[len(itemClocks) // 2]),
average_clock=float(sum(itemClocks) / len(itemClocks)),
min_clock=float(min(itemClocks)),
max_clock=float(max(itemClocks)),
total_wait=float(sum(itemWaits)),
median_wait=float(itemWaits[len(itemWaits) // 2]),
average_wait=float(sum(itemWaits) / len(itemWaits)),
min_wait=float(min(itemWaits)),
max_wait=float(max(itemWaits)),
total_memory=float(sum(itemMemory)),
median_memory=float(itemMemory[len(itemMemory) // 2]),
average_memory=float(sum(itemMemory) / len(itemMemory)),
min_memory=float(min(itemMemory)),
max_memory=float(max(itemMemory)),
name=itemName)
return element[itemName]
def get(tree: Expando, name: str) -> float:
"""Return a float value attribute NAME from TREE."""
try:
return float(tree.get(name, "nan"))
except ValueError:
return float("nan")