def test_read_history():
contents = '''CompletionDate = 1361297173
EnteredCurrentStatus = 1361297000
StartdPrincipal = "unauthenticated@unmapped/192.168.1.110"
EnteredCurrentStatus = 1361297172
TerminationPending = true
*** Offset = -1 ClusterId = 3 ProcId = 7 Owner = "name" CompletionDate = 1361297173
LeaveJobInQueue = false
EnteredCurrentStatus = 1361298176
ExitCode = 0
ClusterId = 3
CompletionDate = 1361298177
*** Offset = -1 ClusterId = 3 ProcId = 9 Owner = "name" CompletionDate = 1361298177
'''
# this is at the end so there is nothing to read
file = StringIO.StringIO(contents)
ads = list(util.readCondorHistory(file, 1361298177))
assert len(ads) == 0
# this is before the last one
ads = list(util.readCondorHistory(file, 1361298176))
assert len(ads) == 1
# this matches the first one so it only includes the last
ads = list(util.readCondorHistory(file, 1361297173))
assert len(ads) == 1
assert set([
"LeaveJobInQueue", "EnteredCurrentStatus", "CompletionDate",
"ExitCode", "ClusterId"
]) == set(ads[0].ad.keys())
assert "1361298176" == ads[0].ad["EnteredCurrentStatus"]
# this matches all
ads = list(util.readCondorHistory(file, 0))
assert len(ads) == 2
assert set([
"CompletionDate", "EnteredCurrentStatus", "StartdPrincipal",
"TerminationPending"
]) == set(ads[1].ad.keys())
assert set([
"LeaveJobInQueue", "EnteredCurrentStatus", "CompletionDate",
"ExitCode", "ClusterId"
]) == set(ads[0].ad.keys())
# make sure we use the LAST EnteredCurrentStatus (it appears twice in each job in history files)
assert "1361297172" == ads[1].ad["EnteredCurrentStatus"]
# simplify this since the ordering can vary
del ads[1].ad["TerminationPending"]
del ads[1].ad["CompletionDate"]
del ads[1].ad["EnteredCurrentStatus"]
assert 'StartdPrincipal = "unauthenticated@unmapped/192.168.1.110"' == ads[
1].get_text()
def test_read_history():
contents = '''CompletionDate = 1361297173
EnteredCurrentStatus = 1361297000
StartdPrincipal = "unauthenticated@unmapped/192.168.1.110"
EnteredCurrentStatus = 1361297172
TerminationPending = true
*** Offset = -1 ClusterId = 3 ProcId = 7 Owner = "name" CompletionDate = 1361297173
LeaveJobInQueue = false
EnteredCurrentStatus = 1361298176
ExitCode = 0
ClusterId = 3
CompletionDate = 1361298177
*** Offset = -1 ClusterId = 3 ProcId = 9 Owner = "name" CompletionDate = 1361298177
'''
# this is at the end so there is nothing to read
file = StringIO.StringIO(contents)
ads = list(util.readCondorHistory(file, 1361298177))
assert len(ads) == 0
# this is before the last one
ads = list(util.readCondorHistory(file, 1361298176))
assert len(ads) == 1
# this matches the first one so it only includes the last
ads = list(util.readCondorHistory(file, 1361297173))
assert len(ads) == 1
assert set(["LeaveJobInQueue", "EnteredCurrentStatus", "CompletionDate", "ExitCode", "ClusterId"]) == set(ads[0].ad.keys())
assert "1361298176" == ads[0].ad["EnteredCurrentStatus"]
# this matches all
ads = list(util.readCondorHistory(file, 0))
assert len(ads) == 2
assert set(["CompletionDate", "EnteredCurrentStatus", "StartdPrincipal", "TerminationPending"]) == set(ads[1].ad.keys())
assert set(["LeaveJobInQueue", "EnteredCurrentStatus", "CompletionDate", "ExitCode", "ClusterId"]) == set(ads[0].ad.keys())
# make sure we use the LAST EnteredCurrentStatus (it appears twice in each job in history files)
assert "1361297172" == ads[1].ad["EnteredCurrentStatus"]
# simplify this since the ordering can vary
del ads[1].ad["TerminationPending"]
del ads[1].ad["CompletionDate"]
del ads[1].ad["EnteredCurrentStatus"]
assert 'StartdPrincipal = "unauthenticated@unmapped/192.168.1.110"' == ads[1].get_text()
def test_skip_invalid():
'''Tests ability to handle incomplete jobs put in by the SOAP API.
In this case, EnteredCurrentStatus is very early (and in fact all the
times can be equally early).
'''
# note the first job is a poison job--it wouldn't appear in real life,
# but it's here to make sure we don't read too far
contents = '''CompletionDate = 19000
EnteredCurrentStatus = 19000
ProcId = 4
*** Offset = -1 ClusterId = 1 ProcId = 2 Owner = "name" CompletionDate = 19000
CompletionDate = 10000
EnteredCurrentStatus = 10000
ProcId = 2
*** Offset = -1 ClusterId = 1 ProcId = 2 Owner = "name" CompletionDate = 10000
CompletionDate = 0
EnteredCurrentStatus = 8000
ProcId = 1
*** Offset = -1 ClusterId = 1 ProcId = 1 Owner = "name" CompletionDate = 0
CompletionDate = 20000
EnteredCurrentStatus = 20000
ProcId = 3
*** Offset = -1 ClusterId = 1 ProcId = 3 Owner = "name" CompletionDate = 20000
'''
file = StringIO.StringIO(contents)
ads = list(util.readCondorHistory(file, 20000))
assert len(ads) == 0
# if we read the 20000 job, we have to also read the 8000 job,
# even though 8000 < 17000, because 8000 measures possibly
# the start of the job, not the end. But we don't have to read the 10000.
ads = list(util.readCondorHistory(file, 17000))
assert len(ads) == 2
# this requires us to read the whole way, including the false job at the start
ads = list(util.readCondorHistory(file, 9000))
assert len(ads) == 4
def test_skip_invalid():
'''Tests ability to handle incomplete jobs put in by the SOAP API.
In this case, EnteredCurrentStatus is very early (and in fact all the
times can be equally early).
'''
# note the first job is a poison job--it wouldn't appear in real life,
# but it's here to make sure we don't read too far
contents = '''CompletionDate = 19000
EnteredCurrentStatus = 19000
ProcId = 4
*** Offset = -1 ClusterId = 1 ProcId = 2 Owner = "name" CompletionDate = 19000
CompletionDate = 10000
EnteredCurrentStatus = 10000
ProcId = 2
*** Offset = -1 ClusterId = 1 ProcId = 2 Owner = "name" CompletionDate = 10000
CompletionDate = 0
EnteredCurrentStatus = 8000
ProcId = 1
*** Offset = -1 ClusterId = 1 ProcId = 1 Owner = "name" CompletionDate = 0
CompletionDate = 20000
EnteredCurrentStatus = 20000
ProcId = 3
*** Offset = -1 ClusterId = 1 ProcId = 3 Owner = "name" CompletionDate = 20000
'''
file = StringIO.StringIO(contents)
ads = list(util.readCondorHistory(file, 20000))
assert len(ads) == 0
# if we read the 20000 job, we have to also read the 8000 job,
# even though 8000 < 17000, because 8000 measures possibly
# the start of the job, not the end. But we don't have to read the 10000.
ads = list(util.readCondorHistory(file, 17000))
assert len(ads) == 2
# this requires us to read the whole way, including the false job at the start
ads = list(util.readCondorHistory(file, 9000))
assert len(ads) == 4
def getHistoryFromFile(self, completed_since, history_file):
'''Reads from the history file backwards to just get the changes.
Returns the text and the latest non-zero CompletionDate found.'''
f = open(history_file, "rb")
try:
ads = list(util.readCondorHistory(f, completed_since))
jobs = []
max_completion = 0
for job in reversed(ads):
jobs.append(job.get_text())
# CompletionDate may not be specified, or may appear as 0, both of which are ignored
max_completion = max(job.ad.get("CompletionDate", 0), max_completion)
logging.debug("Read %s jobs from history file %s" % (len(jobs), history_file))
return ("\n".join(jobs) + "\n", max_completion)
finally:
f.close()
def getHistoryFromFile(self, completed_since, history_file):
'''Reads from the history file backwards to just get the changes.
Returns the text and the latest non-zero CompletionDate found.'''
f = open(history_file, "rb")
try:
ads = list(util.readCondorHistory(f, completed_since))
jobs = []
max_completion = 0
for job in reversed(ads):
jobs.append(job.get_text())
# CompletionDate may not be specified, or may appear as 0, both of which are ignored
max_completion = max(job.ad.get("CompletionDate", 0),
max_completion)
logging.debug("Read %s jobs from history file %s" %
(len(jobs), history_file))
return ("\n".join(jobs) + "\n", max_completion)
finally:
f.close()
*** Offset = -1 ClusterId = 1 ProcId = 3 Owner = "name" CompletionDate = 20000
'''
file = StringIO.StringIO(contents)
ads = list(util.readCondorHistory(file, 20000))
assert len(ads) == 0
# if we read the 20000 job, we have to also read the 8000 job,
# even though 8000 < 17000, because 8000 measures possibly
# the start of the job, not the end. But we don't have to read the 10000.
ads = list(util.readCondorHistory(file, 17000))
assert len(ads) == 2
# this requires us to read the whole way, including the false job at the start
ads = list(util.readCondorHistory(file, 9000))
assert len(ads) == 4
import sys
if __name__ == "__main__":
files = list(util.readCondorHistory(file(sys.argv[1], "rb"), int(sys.argv[2])))
jobs = []
for ad in reversed(files):
jobs.append(ad.get_text())
print "\n".join(jobs)
EnteredCurrentStatus = 20000
ProcId = 3
*** Offset = -1 ClusterId = 1 ProcId = 3 Owner = "name" CompletionDate = 20000
'''
file = StringIO.StringIO(contents)
ads = list(util.readCondorHistory(file, 20000))
assert len(ads) == 0
# if we read the 20000 job, we have to also read the 8000 job,
# even though 8000 < 17000, because 8000 measures possibly
# the start of the job, not the end. But we don't have to read the 10000.
ads = list(util.readCondorHistory(file, 17000))
assert len(ads) == 2
# this requires us to read the whole way, including the false job at the start
ads = list(util.readCondorHistory(file, 9000))
assert len(ads) == 4
import sys
if __name__ == "__main__":
files = list(
util.readCondorHistory(file(sys.argv[1], "rb"), int(sys.argv[2])))
jobs = []
for ad in reversed(files):
jobs.append(ad.get_text())
print "\n".join(jobs)