def test_breakDictionaryIntoChunks_normal(aDict, chunkSize, expectedSizes):
"""breakDictIntoChunks tests"""
result = list(breakDictionaryIntoChunks(aDict, chunkSize))
assert list(map(len, result)) == expectedSizes
mergedResult = {}
for partialDict in result:
mergedResult.update(partialDict)
assert aDict == mergedResult
def submitTasks(self, transDict, clients):
"""Submit the tasks to an external system, using the taskManager provided
:param dict transIDOPBody: transformation body
:param dict clients: dictionary of client objects
:return: S_OK/S_ERROR
"""
transID = transDict["TransformationID"]
transBody = transDict["Body"]
owner = transDict["Owner"]
ownerGroup = transDict["OwnerGroup"]
ownerDN = transDict["OwnerDN"]
method = "submitTasks"
# Get all tasks to submit
tasksToSubmit = clients["TransformationClient"].getTasksToSubmit(
transID, self.tasksPerLoop)
self._logDebug(
"getTasksToSubmit(%s, %s) return value:" %
(transID, self.tasksPerLoop),
tasksToSubmit,
method=method,
transID=transID,
)
if not tasksToSubmit["OK"]:
self._logError("Failed to obtain tasks:",
tasksToSubmit["Message"],
method=method,
transID=transID)
return tasksToSubmit
tasks = tasksToSubmit["Value"]["JobDictionary"]
if not tasks:
self._logVerbose("No tasks found for submission",
method=method,
transID=transID)
return tasksToSubmit
self._logInfo("Obtained %d tasks for submission" % len(tasks),
method=method,
transID=transID)
# Prepare tasks and submits them, by chunks
chunkSize = self.maxParametricJobs if self.bulkSubmissionFlag else self.tasksPerLoop
for taskDictChunk in breakDictionaryIntoChunks(tasks, chunkSize):
res = self._prepareAndSubmitAndUpdateTasks(transID, transBody,
taskDictChunk, owner,
ownerDN, ownerGroup,
clients)
if not res["OK"]:
return res
self._logVerbose("Submitted %d jobs, bulkSubmissionFlag = %s" %
(len(taskDictChunk), self.bulkSubmissionFlag))
return S_OK()
def submitTasks(self, transIDOPBody, clients):
""" Submit the tasks to an external system, using the taskManager provided
:param dict transIDOPBody: transformation body
:param dict clients: dictionary of client objects
:return: S_OK/S_ERROR
"""
transID = transIDOPBody.keys()[0]
transBody = transIDOPBody[transID]['Body']
owner = transIDOPBody[transID]['Owner']
ownerGroup = transIDOPBody[transID]['OwnerGroup']
ownerDN = transIDOPBody[transID]['OwnerDN']
method = 'submitTasks'
# Get all tasks to submit
tasksToSubmit = clients['TransformationClient'].getTasksToSubmit(transID, self.tasksPerLoop)
self._logDebug("getTasksToSubmit(%s, %s) return value:" % (transID, self.tasksPerLoop), tasksToSubmit,
method=method, transID=transID)
if not tasksToSubmit['OK']:
self._logError("Failed to obtain tasks:", tasksToSubmit['Message'],
method=method, transID=transID)
return tasksToSubmit
tasks = tasksToSubmit['Value']['JobDictionary']
if not tasks:
self._logVerbose("No tasks found for submission",
method=method, transID=transID)
return tasksToSubmit
self._logInfo("Obtained %d tasks for submission" % len(tasks),
method=method, transID=transID)
# Prepare tasks and submits them, by chunks
chunkSize = self.maxParametricJobs if self.bulkSubmissionFlag else self.tasksPerLoop
for taskDictChunk in breakDictionaryIntoChunks(tasks, chunkSize):
res = self._prepareAndSubmitAndUpdateTasks(transID, transBody, taskDictChunk,
owner, ownerDN, ownerGroup,
clients)
if not res['OK']:
return res
self._logVerbose("Submitted %d jobs, bulkSubmissionFlag = %s" % (len(taskDictChunk), self.bulkSubmissionFlag))
return S_OK()
def test_breakDictionaryIntoChunks_normal(aDict, chunkSize, expected):
""" breakDictIntoChunks tests"""
assert list(breakDictionaryIntoChunks(aDict, chunkSize)) == expected