def test_checkAllJob(self):
"""test for DataRecoveryAgent checkAllJobs ....................................................."""
from DIRAC.TransformationSystem.Utilities.JobInfo import JobInfo
# test with additional task dicts
from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo
tInfoMock = Mock(name="tInfoMock", spec=TransformationInfo)
mockJobs = dict([(i, self.getTestMock()) for i in range(11)])
mockJobs[2].pendingRequest = True
mockJobs[3].getJobInformation = Mock(
side_effect=(RuntimeError("ARGJob1"), None))
mockJobs[4].getTaskInfo = Mock(side_effect=(TaskInfoException("ARG1"),
None))
taskDict = True
lfnTaskDict = True
self.dra.checkAllJobs(mockJobs, tInfoMock, taskDict, lfnTaskDict)
self.dra.log.error.assert_any_call(MatchStringWith("+++++ Exception"),
"ARGJob1")
self.dra.log.error.assert_any_call(
MatchStringWith("Skip Task, due to TaskInfoException: ARG1"))
self.dra.log.reset_mock()
# test inputFile None
mockJobs = dict([(i, self.getTestMock(nameID=i)) for i in range(5)])
mockJobs[1].inputFiles = []
mockJobs[1].getTaskInfo = Mock(
side_effect=(TaskInfoException("NoInputFile"), None))
mockJobs[1].tType = "MCSimulation"
tInfoMock.reset_mock()
self.dra.checkAllJobs(mockJobs, tInfoMock, taskDict, lfnTaskDict=True)
self.dra.log.notice.assert_any_call(
MatchStringWith("Failing job hard"))
def test_cleanOutputs(tiFixture, userProxyFixture):
"""DIRAC.TransformationSystem.Utilities.TransformationInfo cleanOutputs..................."""
descList = ["lfnDD1", "lfnDD2", "lfnD1", "lfnD2"]
jobInfo = Mock(spec=JobInfo)
jobInfo.outputFiles = ["lfn1", "lfn2"]
jobInfo.outputFileStatus = ["Exists", "Missing"]
tiFixture.enabled = False
tiFixture._TransformationInfo__findAllDescendants = Mock(return_value=descList)
tiFixture.cleanOutputs(jobInfo)
tiFixture.log.notice.assert_any_call(MatchStringWith("Would have removed these files"))
tiFixture.log.notice.assert_any_call(MatchStringWith("lfn1"))
for _name, args, _kwargs in tiFixture.log.notice.mock_calls:
assert "lfn2" not in str(args)
remMock = Mock(name="remmock")
remMock.removeFile.return_value = S_ERROR("arg")
tiFixture.enabled = True
tiFixture._TransformationInfo__findAllDescendants = Mock(return_value=descList)
with patch("DIRAC.TransformationSystem.Utilities.TransformationInfo.DataManager",
return_value=remMock,
autospec=True):
with pytest.raises(RuntimeError) as re:
tiFixture.cleanOutputs(jobInfo)
assert "Failed to remove LFNs: arg" in str(re)
remMock = Mock(name="remmock")
remMock.removeFile.return_value = S_OK({"Successful": {"lfn1": "OK", "lfn2": "OK"},
"Failed": {"lfnD2": "SomeReason",
"lfnD3": "SomeReason",
"lfnDD2": "SomeOtherReason"}})
tiFixture.enabled = True
with patch("DIRAC.TransformationSystem.Utilities.TransformationInfo.DataManager",
autospec=True,
return_value=remMock):
tiFixture.cleanOutputs(jobInfo)
tiFixture.log.notice.assert_any_call(MatchStringWith("Successfully removed 2 files"))
# nothing to remove
jobInfo = Mock(spec=JobInfo)
jobInfo.outputFiles = []
tiFixture._TransformationInfo__findAllDescendants = Mock(return_value=descList)
tiFixture.cleanOutputs(jobInfo)
tiFixture._TransformationInfo__findAllDescendants.assert_not_called()
# nothing to remove
jobInfo = Mock(spec=JobInfo)
jobInfo.outputFiles = ["lfn1", "lfn2"]
jobInfo.outputFileStatus = ["Missing", "Missing"]
tiFixture._TransformationInfo__findAllDescendants = Mock(return_value=[])
tiFixture.cleanOutputs(jobInfo)
tiFixture._TransformationInfo__findAllDescendants.assert_called_once_with(jobInfo.outputFiles)
def test_treatTransformation2(self):
"""test for DataRecoveryAgent treatTransformation success2.........................................."""
getJobMock = Mock(name="getJobMOck")
getJobMock.getJobs.return_value = (Mock(name="jobsMock"), 50, 50)
tinfoMock = Mock(name="infoMock", return_value=getJobMock)
self.dra.checkAllJobs = Mock()
# catch the printout to check path taken
transInfoDict = dict(TransformationID=1234, TransformationName="TestProd12", Type="MCSimulation",
AuthorDN='/some/cert/owner', AuthorGroup='Test_Prod')
with patch("%s.TransformationInfo" % MODULE_NAME, new=tinfoMock):
self.dra.treatTransformation(1234, transInfoDict) # returns None
self.dra.log.notice.assert_any_call(MatchStringWith("Getting tasks..."))
def test_checkAllJob_2(self):
"""Test where failJobHard fails (via cleanOutputs)."""
from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo
tInfoMock = Mock(name='tInfoMock', spec=TransformationInfo)
mockJobs = dict([(i, self.getTestMock()) for i in xrange(5)])
mockJobs[2].pendingRequest = True
mockJobs[3].getTaskInfo = Mock(side_effect=(TaskInfoException('ARGJob3'), None))
mockJobs[3].inputFiles = []
mockJobs[3].tType = 'MCReconstruction'
self.dra._DataRecoveryAgent__failJobHard = Mock(side_effect=(RuntimeError('ARGJob4'), None), name='FJH')
self.dra.checkAllJobs(mockJobs, tInfoMock, tasksDict=True, lfnTaskDict=True)
mockJobs[3].getTaskInfo.assert_called()
self.dra._DataRecoveryAgent__failJobHard.assert_called()
self.dra.log.error.assert_any_call(MatchStringWith('+++++ Exception'), 'ARGJob4')
self.dra.log.reset_mock()
def test_treatTransformation3(self):
"""test for DataRecoveryAgent treatTransformation skip.............................................."""
getJobMock = Mock(name="getJobMOck")
getJobMock.getJobs.return_value = (Mock(name="jobsMock"), 50, 50)
self.dra.checkAllJobs = Mock()
self.dra.jobCache[1234] = (50, 50)
# catch the printout to check path taken
transInfoDict = dict(TransformationID=1234, TransformationName="TestProd12", Type="TestProd",
AuthorDN='/some/cert/owner', AuthorGroup='Test_Prod')
with patch("%s.TransformationInfo" % MODULE_NAME,
autospec=True,
return_value=getJobMock):
self.dra.treatTransformation(transID=1234, transInfoDict=transInfoDict) # returns None
self.dra.log.notice.assert_called_with(MatchStringWith("Skipping transformation 1234"))
def test_execute(self):
"""test for DataRecoveryAgent execute .........................................................."""
self.dra.treatTransformation = Mock()
self.dra.transformationsToIgnore = [123, 456, 789]
self.dra.jobCache = defaultdict(lambda: (0, 0))
self.dra.jobCache[123] = (10, 10)
self.dra.jobCache[124] = (10, 10)
self.dra.jobCache[125] = (10, 10)
# Eligible fails
self.dra.log.reset_mock()
self.dra.getEligibleTransformations = Mock(
return_value=S_ERROR("outcast"))
res = self.dra.execute()
self.assertFalse(res["OK"])
self.dra.log.error.assert_any_call(ANY, MatchStringWith("outcast"))
self.assertEqual("Failure to get transformations", res["Message"])
d123 = dict(
TransformationID=123,
TransformationName="TestProd123",
Type="MCGeneration",
AuthorDN="/some/cert/owner",
AuthorGroup="Test_Prod",
)
d124 = dict(
TransformationID=124,
TransformationName="TestProd124",
Type="MCGeneration",
AuthorDN="/some/cert/owner",
AuthorGroup="Test_Prod",
)
d125 = dict(
TransformationID=125,
TransformationName="TestProd125",
Type="MCGeneration",
AuthorDN="/some/cert/owner",
AuthorGroup="Test_Prod",
)
# Eligible succeeds
self.dra.log.reset_mock()
self.dra.getEligibleTransformations = Mock(return_value=S_OK({
123: d123,
124: d124,
125: d125
}))
res = self.dra.execute()
self.assertTrue(res["OK"])
self.dra.log.notice.assert_any_call(
MatchStringWith(
"Will ignore the following transformations: [123, 456, 789]"))
self.dra.log.notice.assert_any_call(
MatchStringWith("Ignoring Transformation: 123"))
self.dra.log.notice.assert_any_call(
MatchStringWith("Running over Transformation: 124"))
# Notes To Send
self.dra.log.reset_mock()
self.dra.getEligibleTransformations = Mock(return_value=S_OK({
123: d123,
124: d124,
125: d125
}))
self.dra.notesToSend = "Da hast du deine Karte"
sendmailMock = Mock()
sendmailMock.sendMail.return_value = S_OK("Nice Card")
notificationMock = Mock(return_value=sendmailMock)
with patch("%s.NotificationClient" % MODULE_NAME,
new=notificationMock):
res = self.dra.execute()
self.assertTrue(res["OK"])
self.dra.log.notice.assert_any_call(
MatchStringWith(
"Will ignore the following transformations: [123, 456, 789]"))
self.dra.log.notice.assert_any_call(
MatchStringWith("Ignoring Transformation: 123"))
self.dra.log.notice.assert_any_call(
MatchStringWith("Running over Transformation: 124"))
self.assertNotIn(124, self.dra.jobCache) # was popped
self.assertIn(125, self.dra.jobCache) # was not popped
gLogger.notice("JobCache: %s" % self.dra.jobCache)
# sending notes fails
self.dra.log.reset_mock()
self.dra.notesToSend = "Da hast du deine Karte"
sendmailMock = Mock()
sendmailMock.sendMail.return_value = S_ERROR("No stamp")
notificationMock = Mock(return_value=sendmailMock)
with patch("%s.NotificationClient" % MODULE_NAME,
new=notificationMock):
res = self.dra.execute()
self.assertTrue(res["OK"])
self.assertNotIn(124, self.dra.jobCache) # was popped
self.assertIn(125, self.dra.jobCache) # was not popped
self.dra.log.error.assert_any_call(
MatchStringWith("Cannot send notification mail"), ANY)
self.assertEqual("", self.dra.notesToSend)
