def getJobs(self, statusList=None):
"""Get done and failed jobs.
:param list statusList: optional list of status to find jobs
:returns: 3-tuple of OrderedDict of JobInfo objects, keyed by jobID;
number of Done jobs; number of Failed jobs
"""
done = S_OK([])
failed = S_OK([])
if statusList is None:
statusList = ['Done', 'Failed']
if 'Done' in statusList:
self.log.notice("Getting 'Done' Jobs...")
done = self.__getJobs(["Done"])
if 'Failed' in statusList:
self.log.notice("Getting 'Failed' Jobs...")
failed = self.__getJobs(["Failed"])
done = done['Value']
failed = failed['Value']
jobsUnsorted = {}
for job in done:
jobsUnsorted[int(job)] = JobInfo(job, "Done", self.tID,
self.transType)
for job in failed:
jobsUnsorted[int(job)] = JobInfo(job, "Failed", self.tID,
self.transType)
jobs = OrderedDict(sorted(jobsUnsorted.items(), key=lambda t: t[0]))
self.log.notice("Found %d Done Jobs " % len(done))
self.log.notice("Found %d Failed Jobs " % len(failed))
return jobs, len(done), len(failed)
def test__str__(self,
asserts,
assertNots,
trID=1234,
taID=5678,
otherTasks=False,
pendingRequest=False):
jbi = JobInfo(jobID=123,
status="Failed",
tID=trID,
tType="MCReconstruction")
jbi.pendingRequest = pendingRequest
jbi.otherTasks = otherTasks
gLogger.notice("otherTasks: ", jbi.otherTasks)
jbi.taskID = taID
jbi.inputFiles = ["inputFile"]
jbi.inputFilesExist = [True]
jbi.transFileStatus = ["Assigned"]
jbi.outputFiles = ["outputFile"]
jbi.errorCounts = [0]
info = str(jbi)
for assertStr in asserts:
self.assertIn(assertStr, info)
for assertStr in assertNots:
self.assertNotIn(assertStr, info)
def test_failHard(self,
infoCalls,
jID=666,
jStat="Done",
inFiles=None,
ofStat=["Exists"]):
"""Test the job.failHard function."""
from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo
from DIRAC.TransformationSystem.Utilities.JobInfo import JobInfo
tInfoMock = Mock(name="tInfoMock", spec=TransformationInfo)
tInfoMock.reset_mock()
testJob = JobInfo(jobID=666,
status=jStat,
tID=123,
tType="MCSimulation")
testJob.outputFiles = ["/my/stupid/file.lfn"]
testJob.outputFileStatus = ofStat
testJob.otherTasks = True
testJob.inputFiles = inFiles
testJob.inputFileExists = True
testJob.fileStatus = "Processed"
self.dra.inputFilesProcessed = set()
self.dra._DataRecoveryAgent__failJobHard(testJob, tInfoMock) # pylint: disable=protected-access, no-member
gLogger.notice("Expecting calls", infoCalls)
gLogger.notice("Called", tInfoMock.method_calls)
assert len(infoCalls) == len(tInfoMock.method_calls)
for index, infoCall in enumerate(infoCalls):
self.assertIn(infoCall, tInfoMock.method_calls[index])
if jStat == "Done":
self.assertIn("Failing job %s" % jID, self.dra.notesToSend)
else:
self.assertNotIn("Failing job %s" % jID, self.dra.notesToSend)
def test_checkJob_others_(self, counter, infoCalls, jID=1234567, jStat='Done', others=False,
inFiles=['/my/inputfile.lfn'], outFiles=['/my/stupid/file.lfn'],
ifStat=[], ofStat=['Exists'], ifProcessed=[],
tFiStat=['Processed'], errorCount=[]):
from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo
from DIRAC.TransformationSystem.Utilities.JobInfo import JobInfo
tInfoMock = Mock(name="tInfoMock", spec=TransformationInfo)
testJob = JobInfo(jobID=jID, status=jStat, tID=123, tType="MCSimulation")
testJob.outputFiles = outFiles
testJob.outputFileStatus = ofStat
testJob.otherTasks = others
testJob.inputFiles = inFiles
testJob.inputFileStatus = ifStat
testJob.transFileStatus = tFiStat
testJob.errorCounts = errorCount
self.dra.inputFilesProcessed = set(ifProcessed)
self.dra.checkJob(testJob, tInfoMock)
gLogger.notice('Testing counter', counter)
gLogger.notice('Expecting calls', infoCalls)
gLogger.notice('Called', tInfoMock.method_calls)
assert len(infoCalls) == len(tInfoMock.method_calls)
for index, infoCall in enumerate(infoCalls):
self.assertIn(infoCall, tInfoMock.method_calls[index])
for count in range(15):
gLogger.notice('Checking Counter:', count)
if count == counter:
self.assertEqual(self.dra.todo['InputFiles'][count]['Counter'], 1)
else:
self.assertEqual(self.dra.todo['InputFiles'][count]['Counter'], 0)
if 0 <= counter <= 2:
assert set(testJob.inputFiles).issubset(self.dra.inputFilesProcessed)
def test_checkJob(self):
"""test for DataRecoveryAgent checkJob No inputFiles............................................."""
from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo
tInfoMock = Mock(name="tInfoMock", spec=TransformationInfo)
from DIRAC.TransformationSystem.Utilities.JobInfo import JobInfo
# Test First option for MCGeneration
tInfoMock.reset_mock()
testJob = JobInfo(jobID=1234567,
status="Failed",
tID=123,
tType="MCGeneration")
testJob.outputFiles = ["/my/stupid/file.lfn"]
testJob.outputFileStatus = ["Exists"]
self.dra.checkJob(testJob, tInfoMock)
self.assertIn("setJobDone", tInfoMock.method_calls[0])
self.assertEqual(self.dra.todo["NoInputFiles"][0]["Counter"], 1)
self.assertEqual(self.dra.todo["NoInputFiles"][1]["Counter"], 0)
# Test Second option for MCGeneration
tInfoMock.reset_mock()
testJob.status = "Done"
testJob.outputFileStatus = ["Missing"]
self.dra.checkJob(testJob, tInfoMock)
self.assertIn("setJobFailed", tInfoMock.method_calls[0])
self.assertEqual(self.dra.todo["NoInputFiles"][0]["Counter"], 1)
self.assertEqual(self.dra.todo["NoInputFiles"][1]["Counter"], 1)
# Test Second option for MCGeneration
tInfoMock.reset_mock()
testJob.status = "Done"
testJob.outputFileStatus = ["Exists"]
self.dra.checkJob(testJob, tInfoMock)
self.assertEqual(tInfoMock.method_calls, [])
self.assertEqual(self.dra.todo["NoInputFiles"][0]["Counter"], 1)
self.assertEqual(self.dra.todo["NoInputFiles"][1]["Counter"], 1)
def setUp(self):
self.jbi = JobInfo(jobID=123,
status="Failed",
tID=1234,
tType="MCReconstruction")
self.diracAPI = Mock(name="dilcMock",
spec=DIRAC.Interfaces.API.Dirac.Dirac)
self.jobMon = Mock(name="jobMonMock",
spec=DIRAC.WorkloadManagementSystem.Client.
JobMonitoringClient.JobMonitoringClient)
self.jobMon.getInputData = Mock(return_value=S_OK([]))
self.jobMon.getJobAttribute = Mock(return_value=S_OK("0"))
self.jobMon.getJobParameter = Mock(return_value=S_OK({}))
self.diracAPI.getJobJDL = Mock()
self.jdl2 = {
"LogTargetPath":
"/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/00006326_015.tar",
"Executable":
"dirac-jobexec",
"TaskID":
15,
"SoftwareDistModule":
"ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
"JobName":
"00006326_00000015",
"Priority":
1,
"Platform":
"x86_64-slc5-gcc43-opt",
"JobRequirements": {
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"VirtualOrganization":
"ilc",
"Setup":
"ILC-Production",
"CPUTime":
300000,
"OwnerGroup":
"ilc_prod",
"Platforms": [
"x86_64-slc6-gcc44-opt",
"x86_64-slc5-gcc43-opt",
"slc5_amd64_gcc43",
"Linux_x86_64_glibc-2.12",
"Linux_x86_64_glibc-2.5",
],
"UserPriority":
1,
"Sites": [
"LCG.LAPP.fr",
"LCG.UKI-SOUTHGRID-RALPP.uk",
],
"BannedSites":
"LCG.KEK.jp",
"JobTypes":
"MCReconstruction_Overlay",
},
"Arguments":
"jobDescription.xml -o LogLevel=verbose",
"SoftwarePackages": [
"overlayinput.1",
"marlin.v0111Prod",
],
"DebugLFNs":
"",
"Status":
"Created",
"InputDataModule":
"DIRAC.WorkloadManagementSystem.Client.InputDataResolution",
"BannedSites":
"LCG.KEK.jp",
"LogLevel":
"verbose",
"InputSandbox": [
"jobDescription.xml",
"SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/5d3/92f/5d392f5266a796018ab6774ef84cbd31.tar.bz2",
],
"OwnerName":
"sailer",
"StdOutput":
"std.out",
"JobType":
"MCReconstruction_Overlay",
"GridEnv":
"/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
"TransformationID":
6326,
"DIRACSetup":
"ILC-Production",
"StdError":
"std.err",
"IS_PROD":
"True",
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"JobGroup":
0o0006326,
"OutputSandbox": [
"std.err",
"std.out",
],
"JobID":
15756436,
"VirtualOrganization":
"ilc",
"ProductionOutputData": [
"/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/000/yyveyx_o_rec_6326_15.slcio",
"/ilc/prod/clic/500gev/yyveyx_o/ILD/DST/00006326/000/yyveyx_o_dst_6326_15.slcio",
],
"Site":
"ANY",
"OwnerGroup":
"ilc_prod",
"Owner":
"sailer",
"LogFilePath":
"/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/000",
"InputData":
"/ilc/prod/clic/500gev/yyveyx_o/ILD/SIM/00006325/000/yyveyx_o_sim_6325_17.slcio",
}
self.jdlBrokenContent = {
"LogTargetPath":
"/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/00006326_015.tar",
"Executable":
"dirac-jobexec",
"TaskID":
"muahahaha",
"SoftwareDistModule":
"ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
"JobName":
"00006326_00000015",
"Priority":
1,
"Platform":
"x86_64-slc5-gcc43-opt",
"JobRequirements": {
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"VirtualOrganization":
"ilc",
"Setup":
"ILC-Production",
"CPUTime":
300000,
"OwnerGroup":
"ilc_prod",
"Platforms": [
"x86_64-slc6-gcc44-opt",
"x86_64-slc5-gcc43-opt",
"slc5_amd64_gcc43",
"Linux_x86_64_glibc-2.12",
"Linux_x86_64_glibc-2.5",
],
"UserPriority":
1,
"Sites": [
"LCG.LAPP.fr",
"LCG.UKI-SOUTHGRID-RALPP.uk",
],
"BannedSites":
"LCG.KEK.jp",
"JobTypes":
"MCReconstruction_Overlay",
},
"Arguments":
"jobDescription.xml -o LogLevel=verbose",
"SoftwarePackages": [
"overlayinput.1",
"marlin.v0111Prod",
],
"DebugLFNs":
"",
"Status":
"Created",
"InputDataModule":
"DIRAC.WorkloadManagementSystem.Client.InputDataResolution",
"BannedSites":
"LCG.KEK.jp",
"LogLevel":
"verbose",
"InputSandbox": [
"jobDescription.xml",
"SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/5d3/92f/5d392f5266a796018ab6774ef84cbd31.tar.bz2",
],
"OwnerName":
"sailer",
"StdOutput":
"std.out",
"JobType":
"MCReconstruction_Overlay",
"GridEnv":
"/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
"TransformationID":
6326,
"DIRACSetup":
"ILC-Production",
"StdError":
"std.err",
"IS_PROD":
"True",
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"JobGroup":
0o0006326,
"OutputSandbox": [
"std.err",
"std.out",
],
"JobID":
15756436,
"VirtualOrganization":
"ilc",
"ProductionOutputData": [
"/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/000/yyveyx_o_rec_6326_15.slcio",
"/ilc/prod/clic/500gev/yyveyx_o/ILD/DST/00006326/000/yyveyx_o_dst_6326_15.slcio",
],
"Site":
"ANY",
"OwnerGroup":
"ilc_prod",
"Owner":
"sailer",
"LogFilePath":
"/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/000",
"InputData":
"/ilc/prod/clic/500gev/yyveyx_o/ILD/SIM/00006325/000/yyveyx_o_sim_6325_17.slcio",
}
# jdl with single outputdata,
self.jdl1 = {
"LogTargetPath":
"/ilc/prod/clic/3tev/e1e1_o/SID/SIM/00006301/LOG/00006301_10256.tar",
"Executable":
"dirac-jobexec",
"TaskID":
10256,
"SoftwareDistModule":
"ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
"JobName":
"00006301_00010256",
"Priority":
1,
"Platform":
"x86_64-slc5-gcc43-opt",
"JobRequirements": {
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"VirtualOrganization":
"ilc",
"Setup":
"ILC-Production",
"CPUTime":
300000,
"OwnerGroup":
"ilc_prod",
"Platforms": [
"x86_64-slc6-gcc44-opt",
"x86_64-slc5-gcc43-opt",
"slc5_amd64_gcc43",
"Linux_x86_64_glibc-2.12",
"Linux_x86_64_glibc-2.5",
],
"UserPriority":
1,
"Sites": [
"LCG.LAPP.fr",
"LCG.UKI-SOUTHGRID-RALPP.uk",
],
"BannedSites": [
"OSG.MIT.us",
"OSG.SPRACE.br",
],
"JobTypes":
"MCSimulation",
},
"Arguments":
"jobDescription.xml -o LogLevel=verbose",
"SoftwarePackages":
"slic.v2r9p8",
"DebugLFNs":
"",
"Status":
"Created",
"InputDataModule":
"DIRAC.WorkloadManagementSystem.Client.InputDataResolution",
"BannedSites": [
"OSG.MIT.us",
"OSG.SPRACE.br",
],
"LogLevel":
"verbose",
"InputSandbox": [
"jobDescription.xml",
"SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/042/d64/042d64cb0fe73720cbd114a73506c582.tar.bz2",
],
"OwnerName":
"sailer",
"StdOutput":
"std.out",
"JobType":
"MCSimulation",
"GridEnv":
"/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
"TransformationID":
6301,
"DIRACSetup":
"ILC-Production",
"StdError":
"std.err",
"IS_PROD":
"True",
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"JobGroup":
"00006301",
"OutputSandbox": [
"std.err",
"std.out",
],
"JobID":
15756456,
"VirtualOrganization":
"ilc",
"ProductionOutputData":
"/ilc/prod/clic/3tev/e1e1_o/SID/SIM/00006301/010/e1e1_o_sim_6301_10256.slcio",
"Site":
"ANY",
"OwnerGroup":
"ilc_prod",
"Owner":
"sailer",
"LogFilePath":
"/ilc/prod/clic/3tev/e1e1_o/SID/SIM/00006301/LOG/010",
"InputData":
"/ilc/prod/clic/3tev/e1e1_o/gen/00006300/004/e1e1_o_gen_6300_4077.stdhep",
}
self.jdlNoInput = {
"LogTargetPath":
"/ilc/prod/clic/1.4tev/ea_qqqqnu/gen/00006498/LOG/00006498_1307.tar",
"Executable":
"dirac-jobexec",
"TaskID":
1307,
"SoftwareDistModule":
"ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
"JobName":
"00006498_00001307",
"Priority":
1,
"Platform":
"x86_64-slc5-gcc43-opt",
"JobRequirements": {
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"VirtualOrganization":
"ilc",
"Setup":
"ILC-Production",
"CPUTime":
300000,
"OwnerGroup":
"ilc_prod",
"Platforms": [
"x86_64-slc6-gcc44-opt",
"x86_64-slc5-gcc43-opt",
"slc5_amd64_gcc43",
"Linux_x86_64_glibc-2.12",
"Linux_x86_64_glibc-2.5",
],
"UserPriority":
1,
"BannedSites":
"LCG.KEK.jp",
"JobTypes":
"MCGeneration",
},
"Arguments":
"jobDescription.xml -o LogLevel=verbose",
"SoftwarePackages":
"whizard.SM_V57",
"DebugLFNs":
"",
"Status":
"Created",
"InputDataModule":
"DIRAC.WorkloadManagementSystem.Client.InputDataResolution",
"BannedSites":
"LCG.KEK.jp",
"LogLevel":
"verbose",
"InputSandbox": [
"jobDescription.xml",
"SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/b2a/d98/b2ad98c3e240361a4253c4bb277be478.tar.bz2",
],
"OwnerName":
"sailer",
"StdOutput":
"std.out",
"JobType":
"MCGeneration",
"GridEnv":
"/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
"TransformationID":
6498,
"DIRACSetup":
"ILC-Production",
"StdError":
"std.err",
"IS_PROD":
"True",
"OwnerDN":
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer",
"JobGroup":
"00006498",
"OutputSandbox": [
"std.err",
"std.out",
],
"JobID":
15762268,
"VirtualOrganization":
"ilc",
"ProductionOutputData":
"/ilc/prod/clic/1.4tev/ea_qqqqnu/gen/00006498/001/ea_qqqqnu_gen_6498_1307.stdhep",
"Site":
"ANY",
"OwnerGroup":
"ilc_prod",
"Owner":
"sailer",
"LogFilePath":
"/ilc/prod/clic/1.4tev/ea_qqqqnu/gen/00006498/LOG/001",
"InputData":
"",
}