def test_parse_next(self):
tf = tempfile.TemporaryFile()
tf.write("[foo = 1] [bar = 2]")
tf.seek(0)
ad = classad.parseNext(tf)
self.assertEqual(len(ad), 1)
self.assertEqual(ad["foo"], 1)
self.assertEquals(" [bar = 2]", tf.read())
tf = tempfile.TemporaryFile()
tf.write("-----\nfoo = 1\n\nbar = 2\n")
tf.seek(0)
ad = classad.parseNext(tf)
self.assertEqual(len(ad), 1)
self.assertEqual(ad["foo"], 1)
self.assertEquals("bar = 2\n", tf.read())
def test_parse_next(self):
tf = tempfile.TemporaryFile()
tf.write("[foo = 1] [bar = 2]")
tf.seek(0)
ad = classad.parseNext(tf)
self.assertEqual(len(ad), 1)
self.assertEqual(ad["foo"], 1)
self.assertEquals(" [bar = 2]", tf.read())
tf = tempfile.TemporaryFile()
tf.write("-----\nfoo = 1\n\nbar = 2\n")
tf.seek(0)
ad = classad.parseNext(tf)
self.assertEqual(len(ad), 1)
self.assertEqual(ad["foo"], 1)
self.assertEquals("bar = 2\n", tf.read())
def parse_classad(classad_file):
all_ads = []
current_cluster_id = 0
while True:
try:
ad = classad.parseNext(classad_file)
except StopIteration:
# We are done
break
if 'ClusterId' in ad and ad['ClusterId'] != current_cluster_id:
current_cluster_id = ad['ClusterId']
# create ad objects, add to the output, and keep updating the objects
cluster_ad = classad.ClassAd()
proc_ads = []
all_ads.append((cluster_ad, proc_ads))
# Adjust ad contents
del ad['ClusterId']
del ad['ProcId']
if 'x509userproxy' in ad:
# condor_submit dry-run / dump do not evaluate the proxy contents
set_x509_attributes(ad)
if len(cluster_ad) == 0:
cluster_ad.update(ad)
proc_ads.append((ad, 1))
return all_ads
def test_parse_next(self):
tf = tempfile.TemporaryFile()
tf.write(b"[foo = 1] [bar = 2]")
tf.seek(0)
if sys.version_info > (3, ):
tf, tf_ = open(tf.fileno()), tf
ad = classad.parseNext(tf)
self.assertEqual(len(ad), 1)
self.assertEqual(ad["foo"], 1)
self.assertEqual(" [bar = 2]", tf.read())
tf = tempfile.TemporaryFile()
tf.write(b"-----\nfoo = 1\n\nbar = 2\n")
tf.seek(0)
if sys.version_info > (3, ):
tf, tf_ = open(tf.fileno()), tf
ad = classad.parseNext(tf)
self.assertEqual(len(ad), 1)
self.assertEqual(ad["foo"], 1)
self.assertEqual("bar = 2\n", tf.read())
def test_old_classad_v2(self):
ad = classad.parseNext(open("tests/test.old.ad"))
contents = open("tests/test.old.ad").read()
keys = []
for line in contents.splitlines():
info = line.split(" = ")
if len(info) != 2:
continue
self.assertTrue(info[0] in ad)
self.assertEqual(ad.lookup(info[0]).__repr__(), info[1])
keys.append(info[0])
for key in ad:
self.assertTrue(key in keys)
def test_old_classad_v2(self):
ad = classad.parseNext(open("tests/test.old.ad"))
contents = open("tests/test.old.ad").read()
keys = []
for line in contents.splitlines():
info = line.split(" = ")
if len(info) != 2:
continue
self.assertTrue(info[0] in ad)
self.assertEqual(ad.lookup(info[0]).__repr__(), info[1])
keys.append(info[0])
for key in ad:
self.assertTrue(key in keys)
def test_can_parse_ads_across_pipes(ad_string, parser):
"""
The parser must be set manually. Auto-discovery won't work on a pipe because
it can't rewind using seek.
"""
r, w = os.pipe()
with open(w, mode="w") as wf:
wf.write(ad_string)
with open(r, mode="r") as rf:
ad = classad.parseNext(rf, parser)
assert ad["foo"] == 1
def test_pipes(self):
# One regression we saw in the ClassAd library is the new
# parsing routines would fail if tell/seek was non-functional.
r, w = os.pipe()
rfd = os.fdopen(r, 'r')
wfd = os.fdopen(w, 'w')
wfd.write("[foo = 1]")
wfd.close()
ad = classad.parseNext(rfd ,parser=classad.Parser.New)
self.assertEquals(tuple(dict(ad).items()), (('foo', 1),))
self.assertRaises(StopIteration, classad.parseNext, rfd, classad.Parser.New)
rfd.close()
r, w = os.pipe()
rfd = os.fdopen(r, 'r')
wfd = os.fdopen(w, 'w')
wfd.write("[foo = 1]")
wfd.close()
self.assertRaises(ValueError, classad.parseNext, rfd)
rfd.close()
def test_pipes(self):
# One regression we saw in the ClassAd library is the new
# parsing routines would fail if tell/seek was non-functional.
r, w = os.pipe()
rfd = os.fdopen(r, 'r')
wfd = os.fdopen(w, 'w')
wfd.write("[foo = 1]")
wfd.close()
ad = classad.parseNext(rfd, parser=classad.Parser.New)
self.assertEqual(tuple(dict(ad).items()), (('foo', 1), ))
self.assertRaises(StopIteration, classad.parseNext, rfd,
classad.Parser.New)
rfd.close()
r, w = os.pipe()
rfd = os.fdopen(r, 'r')
wfd = os.fdopen(w, 'w')
wfd.write("[foo = 1]")
wfd.close()
self.assertRaises(ValueError, classad.parseNext, rfd)
rfd.close()
# close all the opening files
job_ID_history.close()
condor_history_log.close()
# parse the generated condor history log file and evaluate attributes
# write to new log files
input = open("/home/bockelman/zzhang/ELK_stack/condor_history_log_backup/" + \
"condor_history_" + timestamp_str + ".log", "r")
output = open("/var/log/condor_history/p_condor_history_" + \
timestamp_str + ".log", "a+")
global_job_id_set = set()
while True:
try:
ad = classad.parseNext(input)
for k in ad:
if ad.eval(k) is classad.Value.Undefined:
del ad[k]
else:
ad[k] = ad.eval(k)
if "GlobalJobId" in ad.keys():
if ad["GlobalJobId"] in global_job_id_set:
pass
else:
global_job_id_set.add(ad["GlobalJobId"])
output.write(ad.printOld()+"\n")
else:
output.write(ad.printOld()+"\n")
except StopIteration:
break
job_ID_history = open(checkpoint_path, "w")
job_ID_history.write(output)
# close all the opening files
job_ID_history.close()
condor_history_log.close()
# parse the generated condor history log file and evaluate attributes
# write to new log files
input = open(raw_log, "r")
output = open(processed_log, "a+")
global_job_id_set = set()
while True:
try:
ad = classad.parseNext(input)
for k in ad:
if ad.eval(k) is classad.Value.Undefined:
del ad[k]
else:
ad[k] = ad.eval(k)
if "GlobalJobId" in ad.keys():
if ad["GlobalJobId"] in global_job_id_set:
pass
else:
global_job_id_set.add(ad["GlobalJobId"])
check_data_locality(ad)
output.write(ad.printOld()+"\n")
else:
check_data_locality(ad)
output.write(ad.printOld()+"\n")
def test_parse_next_ad_from_file_like_object(ad_file):
with ad_file.open(mode="r") as f:
ads = [classad.parseNext(f), classad.parseNext(f)]
assert ads[0]["foo"] == "bar"
assert ads[1]["foo"] == "wiz"
def test_parse_next_ad_from_string(ad_string):
ads = [classad.parseNext(ad_string), classad.parseNext(ad_string)]
assert ads[0]["foo"] == "bar"
assert ads[1]["foo"] == "wiz"
