def mergejobfiles(jobfiles,
outputfile,
checkids=False,
ignorejobsidentifier=False):
if type(jobfiles) == str:
jobfiles = [jobfiles]
root = lxml.Element("jobs")
for jobfile in jobfiles:
numberofjobs = 0
ignoredjobs = 0
smallroot = XmlParser(jobfile)
for entry in smallroot.iter('job'):
if ignorejobsidentifier != False:
status = entry.find("status").text
if status in ignorejobsidentifier:
ignoredjobs += 1
continue
numberofjobs += 1
if checkids:
jobid = int(entry.find("id").text)
for entry in root.iter('job'):
jid = int(entry.find("id").text)
if jobid == jid:
print "Job {} from {} has the same ID as a job parsed before. Exiting..".format(
jobid, jobfile)
sys.exit()
root.append(entry)
print "Found {} jobs in file {}".format(numberofjobs, jobfile)
if ignorejobsidentifier != False:
if type(ignorejobsidentifier) == str:
ignorejobsidentifier = [ignorejobsidentifier]
print "which are not of type:{}".format(
" ".join(ignorejobsidentifier))
print "Ignored jobs: {}".format(ignoredjobs)
XmlWriter(root, outputfile)
def mergejobfiles(jobfiles,outputfile,checkids=False,ignorejobsidentifier=False):
if type(jobfiles)==str:
jobfiles=[jobfiles]
root=lxml.Element("jobs")
for jobfile in jobfiles:
numberofjobs=0
ignoredjobs=0
smallroot=XmlParser(jobfile)
for entry in smallroot.iter('job'):
if ignorejobsidentifier!=False:
status=entry.find("status").text
if status in ignorejobsidentifier:
ignoredjobs+=1
continue
numberofjobs+=1
if checkids:
jobid=int(entry.find("id").text)
for entry in root.iter('job'):
jid=int(entry.find("id").text)
if jobid==jid:
print "Job {} from {} has the same ID as a job parsed before. Exiting..".format(jobid,jobfile)
sys.exit()
root.append(entry)
print "Found {} jobs in file {}".format(numberofjobs,jobfile)
if ignorejobsidentifier!=False:
if type(ignorejobsidentifier)==str:
ignorejobsidentifier=[ignorejobsidentifier]
print "which are not of type:{}".format(" ".join(ignorejobsidentifier))
print "Ignored jobs: {}".format(ignoredjobs)
XmlWriter(root,outputfile)
def splittjobfile(jobfile, jobfiles, ignorejobsidentifier=False):
root = XmlParser(jobfile)
numberofjobs = 0
ignoredjobs = 0
for entry in root.iter('job'):
if ignorejobsidentifier != False:
status = entry.find("status").text
if status in ignorejobsidentifier:
root.remove(entry)
ignoredjobs += 1
continue
numberofjobs += 1
print "Found {} jobs in in file {}".format(numberofjobs, jobfile)
if ignorejobsidentifier != False:
if type(ignorejobsidentifier) == str:
ignorejobsidentifier = [ignorejobsidentifier]
print "which are not of type:{}".format(" ".join(ignorejobsidentifier))
print "Ignored jobs: {}".format(ignoredjobs)
numberoffiles = len(jobfiles)
jobsperfile = numberofjobs / numberoffiles
remaining = numberofjobs % numberoffiles
jobsperfilelist = []
roots = []
for i in range(numberoffiles):
roots.append(lxml.Element("jobs"))
if remaining > 0:
jobsperfilelist.append(jobsperfile + 1)
remaining -= 1
else:
jobsperfilelist.append(jobsperfile)
i = 0
j = 1
for entry in root.iter('job'):
if j > jobsperfilelist[i]:
i += 1
j = 1
j += 1
roots[i].append(entry)
print "Split up jobfile into {} files with".format(numberoffiles)
print "with: {} jobs each".format(" ".join(map(str, jobsperfilelist)))
for outputfile, root in zip(jobfiles, roots):
XmlWriter(root, outputfile)
def splittjobfile(jobfile,jobfiles,ignorejobsidentifier=False):
root=XmlParser(jobfile)
numberofjobs=0
ignoredjobs=0
for entry in root.iter('job'):
if ignorejobsidentifier!=False:
status=entry.find("status").text
if status in ignorejobsidentifier:
root.remove(entry)
ignoredjobs+=1
continue
numberofjobs+=1
print "Found {} jobs in in file {}".format(numberofjobs,jobfile)
if ignorejobsidentifier!=False:
if type(ignorejobsidentifier)==str:
ignorejobsidentifier=[ignorejobsidentifier]
print "which are not of type:{}".format(" ".join(ignorejobsidentifier))
print "Ignored jobs: {}".format(ignoredjobs)
numberoffiles=len(jobfiles)
jobsperfile=numberofjobs/numberoffiles
remaining=numberofjobs%numberoffiles
jobsperfilelist=[]
roots=[]
for i in range(numberoffiles):
roots.append(lxml.Element("jobs"))
if remaining>0:
jobsperfilelist.append(jobsperfile+1)
remaining-=1
else:
jobsperfilelist.append(jobsperfile)
i=0
j=1
for entry in root.iter('job'):
if j>jobsperfilelist[i]:
i+=1
j=1
j+=1
roots[i].append(entry)
print "Split up jobfile into {} files with".format(numberoffiles)
print "with: {} jobs each".format(" ".join(map(str,jobsperfilelist)))
for outputfile,root in zip(jobfiles,roots):
XmlWriter(root,outputfile)
def readexcitonxml_egwbse(filename):
results = []
root = XmlParser(filename)
for job in root.iter('job'):
output = job.find("output")
segment = output.find("segment")
gwbse = segment.find("GWBSE")
mol = readexcitonxml_molecule(gwbse)
mol.setId(int(segment.get("id")))
mol.setName(segment.get("type"))
results.append(mol)
return results
def resetjobfile(jobfile,assigned=True,failed=True):
root=XmlParser(jobfile)
piece=""
if assigned and failed:
piece="ASSIGNED and FAILED"
elif assigned:
piece="ASSIGNED"
elif failed:
piece="FAILED"
else:
piece="No Entry"
print "Resetting for file {}: {} to AVAILABLE".format(jobfile,piece)
for entry in root.iter('job'):
status=entry.find("status").text
if assigned==True and status=="ASSIGNED":
entry.find("status").text="AVAILABLE"
elif failed==True and status=="FAILED":
entry.find("status").text="AVAILABLE"
XmlWriter(root,jobfile)
def resetjobfile(jobfile, assigned=True, failed=True, complete=False):
root = XmlParser(jobfile)
piece = ""
if assigned:
piece += "ASSIGNED "
if failed:
piece += "FAILED "
if complete:
piece += "COMPLETE "
if piece == "":
piece = "No Entry"
print "Resetting for file {}: {} to AVAILABLE".format(jobfile, piece)
for entry in root.iter('job'):
status = entry.find("status").text
if assigned == True and status == "ASSIGNED":
entry.find("status").text = "AVAILABLE"
elif failed == True and status == "FAILED":
entry.find("status").text = "AVAILABLE"
elif complete == True and status == "COMPLETE":
entry.find("status").text = "AVAILABLE"
XmlWriter(root, jobfile)
def infojobfile(jobfile):
complete=0
available=0
assigned=0
failed=0
root=XmlParser(jobfile)
for entry in root.iter('job'):
status=entry.find("status").text
if status=="ASSIGNED":
assigned+=1
elif status=="AVAILABLE":
available+=1
elif status=="COMPLETE":
complete+=1
elif status=="FAILED":
failed+=1
else:
jobid=entry.find("id").text
print "WARNING: Job status {} for job id:{} in file {} not known".format(status,jobid,jobfile)
total=complete+available+assigned+failed
return total,complete,available,assigned,failed
def infojobfile(jobfile):
complete = 0
available = 0
assigned = 0
failed = 0
root = XmlParser(jobfile)
for entry in root.iter('job'):
status = entry.find("status").text
if status == "ASSIGNED":
assigned += 1
elif status == "AVAILABLE":
available += 1
elif status == "COMPLETE":
complete += 1
elif status == "FAILED":
failed += 1
else:
jobid = entry.find("id").text
print "WARNING: Job status {} for job id:{} in file {} not known".format(
status, jobid, jobfile)
total = complete + available + assigned + failed
return total, complete, available, assigned, failed
from __tools__ import MyParser
from __tools__ import XmlParser
from __tools__ import XmlWriter
import lxml.etree as lxml
parser = MyParser(description="Delete Entries from Jobfile")
parser.add_argument("--jobfile", "-j", type=str, required=True, help="jobfile")
parser.add_argument("--output",
"-o",
type=str,
default="joboutput.xml",
help="jobfile")
parser.add_argument("--exclude",
type=str,
nargs="+",
help="Tags to exclude from jobfile e,h,n,s,t")
args = parser.parse_args()
print "Removing states with tags including: {} ".format(" ".join(args.exclude))
root = XmlParser(args.jobfile)
for entry in root.iter('job'):
status = entry.find("tag").text
for eex in args.exclude:
if eex in status:
root.remove(entry)
break
print "Writing to {}".format(args.output)
XmlWriter(root, args.output)
if i + 1 not in args.id:
continue
if args.compare:
segtype = segment.get('type')
if segtype not in args.format:
continue
segid = segment.get("id")
mpsfile = os.path.join(args.path, (args.format).format(segid))
segment.set('mps_file', mpsfile)
print "writing"
with open(args.jobfile, 'w') as f:
f.write(lxml.tostring(root, pretty_print=True))
elif filetype == "xml":
root = XmlParser(args.jobfile)
for job in root.iter('job'):
inputs = job.find('input')
tags = job.find("tag")
tags = tags.text.split(":")
segid = int(tags[0])
name = tags[1]
state = tags[2]
if state == args.state:
mpsfile = os.path.join(args.path, (args.format).format(segid))
mpsfile = "{}:{}:{}".format(segid, name, mpsfile)
inputs.text = mpsfile
with open(args.jobfile, 'w') as f:
f.write(lxml.tostring(root, pretty_print=True))
elif filetype == "tab":
if i+1 not in args.id:
continue
if args.compare:
segtype=segment.get('type')
if segtype not in args.format:
continue
segid=segment.get("id")
mpsfile=os.path.join(args.path,(args.format).format(segid))
segment.set('mps_file',mpsfile)
print "writing"
with open(args.jobfile, 'w') as f:
f.write(lxml.tostring(root, pretty_print=True))
elif filetype=="xml":
root=XmlParser(args.jobfile)
for job in root.iter('job'):
inputs=job.find('input')
tags=job.find("tag")
tags=tags.text.split(":")
segid=int(tags[0])
name=tags[1]
state=tags[2]
if state==args.state:
mpsfile=os.path.join(args.path,(args.format).format(segid))
mpsfile="{}:{}:{}".format(segid,name,mpsfile)
inputs.text=mpsfile
with open(args.jobfile, 'w') as f:
f.write(lxml.tostring(root, pretty_print=True))
