def setUp(self):
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-groupuser-interface", "GroupUser")
#create a test owner for the collections in this test
self.owner1 = makeUser("DMWM", "evansde77", self.testInit.couchUrl, self.testInit.couchDbName)
self.owner1.connect()
self.owner1.create()
self.owner2 = makeUser("DMWM", "drsm79", self.testInit.couchUrl, self.testInit.couchDbName)
self.owner2.connect()
self.owner2.create()
def setUp(self):
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-groupuser-interface", "GroupUser")
#create a test owner for the collections in this test
self.owner1 = makeUser("DMWM", "evansde77", self.testInit.couchUrl, self.testInit.couchDbName)
self.owner1.connect()
self.owner1.create()
self.owner2 = makeUser("DMWM", "drsm79", self.testInit.couchUrl, self.testInit.couchDbName)
self.owner2.connect()
self.owner2.create()
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
EmulatorHelper.setEmulators(phedex=False,
dbs=False,
siteDB=True,
requestMgr=False)
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
self.siteWhitelist = ['T2_XX_SiteA']
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl,
self.acdcDBName)
user.create()
return
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk']
self.siteWhitelist = ['T2_XX_SiteA']
siteDB = SiteDB()
#Convert phedex node name to a valid processing site name
self.PSNs = siteDB.PNNstoPSNs(self.validLocations)
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
user.create()
return
def createUser(self, username):
self.owner = makeUser(self.group['name'], username,
couchUrl = self.dburl,
couchDatabase = self.dbname)
self.owner.create()
self.owner.ownThis(self.document)
return
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
EmulatorHelper.setEmulators(siteDB = True)
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['srm-cms.gridpp.rl.ac.uk', 'cmssrm.fnal.gov', 'srm.unl.edu']
self.validLocationsCMSNames = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
self.siteWhitelist = ['T2_XX_SiteA']
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl = self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create = False)
user = makeUser('unknown', '*****@*****.**', self.couchUrl, self.acdcDBName)
user.create()
return
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=False)
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
self.siteWhitelist = ['T2_XX_SiteA']
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
user.create()
return
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = [
'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk'
]
self.siteWhitelist = ['T2_XX_SiteA']
cric = CRIC()
# Convert phedex node name to a valid processing site name
self.PSNs = cric.PNNstoPSNs(self.validLocations)
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl,
self.acdcDBName)
user.create()
return
def newOwner(self, group, user):
"""
_newOwner_
Add a new owner
"""
userInstance = makeUser(group, user, self.url, self.database)
userInstance.create()
return userInstance
def newOwner(self, group, user):
"""
_newOwner_
Add a new owner
"""
userInstance = makeUser(group, user, self.url, self.database)
userInstance.create()
return userInstance
def setUp(self):
"""setup couch instance"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-acdc-couchcollection", "GroupUser",
"ACDC")
self.owner = makeUser("DMWM", "evansde77", self.testInit.couchUrl,
self.testInit.couchDbName)
self.owner.connect()
self.owner.create()
def setUp(self):
"""setup couch instance"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-acdc-couchcollection", "GroupUser",
"ACDC")
self.owner = makeUser("DMWM", "evansde77", self.testInit.couchUrl,
self.testInit.couchDbName)
self.owner.connect()
self.owner.create()
def testA(self):
"""instantiation"""
try:
coll = Collection()
except Exception as ex:
msg = "Unable to instantiate Collection with no args"
msg += "\n%s" % str(ex)
self.fail(msg)
user = makeUser("somegroup", "someuser")
coll.setOwner(user)
self.assertEqual(coll.owner, user)
def connectUserGroup(self, groupname, username):
"""
_connectUserGroup_
"""
self.group = Group(name = groupname)
self.group.setCouch(self.dburl, self.dbname)
self.group.connect()
self.owner = makeUser(groupname, username,
couchUrl = self.dburl,
couchDatabase = self.dbname)
return
def listOwners(self):
"""
_listOwners_
List the owners in the DB
"""
result = self.couchdb.loadView("GroupUser", 'name_map', {}, [])
users = []
for row in result[u'rows']:
group = row[u'key'][0]
user = row[u'key'][1]
users.append(makeUser(group, user, self.url, self.database))
return users
def listOwners(self):
"""
_listOwners_
List the owners in the DB
"""
result = self.couchdb.loadView("GroupUser", 'name_map', {}, [])
users = []
for row in result[u'rows']:
group = row[u'key'][0]
user = row[u'key'][1]
users.append(makeUser(group, user, self.url, self.database))
return users
def setUp(self):
"""
_setUp_
Set up couch test environment.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-acdc-couchfileset", "GroupUser",
"ACDC")
self.owner = makeUser("DMWM", "evansde77", self.testInit.couchUrl,
self.testInit.couchDbName)
self.owner.connect()
self.owner.create()
return
def setUp(self):
"""
_setUp_
Set up couch test environment.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-acdc-couchfileset", "GroupUser",
"ACDC")
self.owner = makeUser("DMWM", "evansde77", self.testInit.couchUrl,
self.testInit.couchDbName)
self.owner.connect()
self.owner.create()
return
def defineRequests(workload,
requestInfo,
acdcCouchUrl,
acdcCouchDb,
requestor,
group,
dbsUrl,
datasetInformation=None):
"""
_defineRequests_
This is the ultimate function,
it will create JSONs for the appropiate resubmission requests
that can be feed into the reqmgr.py script and it will assemble
acdc records that can be uploaded to the database.
"""
# First retrieve the run and block lists and load
# the information of all datasets
topTask = workload.getTopLevelTask()[0]
runWhitelist = topTask.inputRunWhitelist()
runBlacklist = topTask.inputRunBlacklist()
blockWhitelist = topTask.inputBlockWhitelist()
blockBlacklist = topTask.inputBlockBlacklist()
inputDataset = workload.listInputDatasets()[0]
outputModules = getOutputModules(workload)
if datasetInformation is None:
datasetInformation = {}
logging.info("Loading DBS information for the datasets...")
datasetInformation[inputDataset] = getFiles(inputDataset, runBlacklist,
runWhitelist,
blockBlacklist,
blockWhitelist, dbsUrl)
for dataset in workload.listOutputDatasets():
datasetInformation[dataset] = getFiles(dataset, runBlacklist,
runWhitelist,
blockBlacklist,
blockWhitelist, dbsUrl)
logging.info("Finished loading DBS information for the datasets...")
# Now get the information about the datasets and tasks
nodes, edges = buildDatasetTree(workload)
logging.info("Dataset tree built...")
for k, v in nodes.items():
logging.debug("%s : %s" % (k, v))
for k, v in edges.items():
logging.debug("%s : %s" % (k, v))
# Load the difference information between input and outputs
differenceInformation = buildDifferenceMap(workload, datasetInformation)
logging.info("Difference map processed...")
logging.debug("%s" % str(differenceInformation))
# Define an object that will hold the potential requests
requests = []
logging.info("Now definining the required requests...")
# First generate requests for the datasets with children, that way we can
# shoot the requests with skims in single requests
for dataset in differenceInformation.keys():
if dataset not in nodes:
continue
datasetsToRecover = [dataset]
diffedLumis = differenceInformation[dataset]
taskToRecover = edges[(inputDataset, dataset)]['task']
outputModulesToRecover = edges[(inputDataset, dataset)]['outMod']
intersectionDiff = {}
for childDataset in nodes[dataset]:
childDiffLumis = differenceInformation[childDataset]
matchAvailable = False
for run in diffedLumis:
if run in childDiffLumis:
for lumi in diffedLumis[run]:
if lumi in childDiffLumis[run]:
matchAvailable = True
break
if matchAvailable:
outputModulesToRecover.extend(edges[(dataset,
childDataset)]['outMod'])
datasetsToRecover.append(childDataset)
for run in diffedLumis:
if run in childDiffLumis:
if run not in intersectionDiff:
intersectionDiff[run] = set()
intersectionDiff[
run] = diffedLumis[run] & childDiffLumis[run]
else:
intersectionDiff[
run] &= diffedLumis[run] & childDiffLumis[run]
else:
intersectionDiff[run] = set()
for run in intersectionDiff:
if not intersectionDiff[run]:
del intersectionDiff[run]
if not intersectionDiff:
# Can't create request for this dataset + children
continue
for run in intersectionDiff:
for childDataset in nodes[dataset]:
childDiffLumis = differenceInformation[childDataset]
if run in childDiffLumis:
childDiffLumis[run] -= intersectionDiff[run]
if not childDiffLumis[run]:
del childDiffLumis[run]
diffedLumis[run] -= intersectionDiff[run]
if not diffedLumis[run]:
del diffedLumis[run]
if not diffedLumis:
del differenceInformation[dataset]
for childDataset in nodes[dataset]:
if not differenceInformation[childDataset]:
del differenceInformation[childDataset]
requestObject = {
'task': taskToRecover,
'input': inputDataset,
'lumis': intersectionDiff,
'outMod': outputModulesToRecover,
'outputs': datasetsToRecover
}
requests.append(requestObject)
# Now go through all the output datasets, creating a single request for
# each
for dataset in differenceInformation:
datasetsToRecover = [dataset]
diffedLumis = differenceInformation[dataset]
if (inputDataset, dataset) in edges:
taskToRecover = edges[(inputDataset, dataset)]['task']
outputModulesToRecover = edges[(inputDataset, dataset)]['outMod']
parentDataset = inputDataset
else:
for parentDataset in nodes:
if dataset in nodes[parentDataset]:
taskToRecover = edges[(parentDataset, dataset)]['task']
outputModulesToRecover = edges[(parentDataset,
dataset)]['outMod']
break
requestObject = {
'task': taskToRecover,
'input': parentDataset,
'lumis': diffedLumis,
'outMod': outputModulesToRecover,
'outputs': datasetsToRecover
}
requests.append(requestObject)
logging.info("About to upload ACDC records to: %s/%s" %
(acdcCouchUrl, acdcCouchDb))
# With the request objects we need to build ACDC records and
# request JSONs
pprint(requests)
for idx, requestObject in enumerate(requests):
collectionName = '%s_%s' % (workload.name(), str(uuid.uuid1()))
filesetName = requestObject['task']
collection = CouchCollection(**{
"url": acdcCouchUrl,
"database": acdcCouchDb,
"name": collectionName
})
owner = makeUser(workload.getOwner()['group'],
workload.getOwner()['name'], acdcCouchUrl,
acdcCouchDb)
collection.setOwner(owner)
files = 0
lumis = 0
for lfn in datasetInformation[requestObject['input']]:
fileInfo = datasetInformation[requestObject['input']][lfn]
fileRuns = {}
for run in fileInfo['runs']:
if run in requestObject['lumis']:
for lumi in fileInfo['runs'][run][0]:
if lumi in requestObject['lumis'][run]:
if run not in fileRuns:
fileRuns[run] = []
fileRuns[run].append(lumi)
lumis += 1
if fileRuns:
files += 1
fileset = CouchFileset(
**{
"url": acdcCouchUrl,
"database": acdcCouchDb,
"name": filesetName
})
fileset.setCollection(collection)
acdcRuns = []
for run in fileRuns:
runObject = {}
runObject['run_number'] = int(run)
runObject['lumis'] = fileRuns[run]
acdcRuns.append(runObject)
acdcFile = {
"lfn": lfn,
"first_event": 0,
"last_event": 0,
"checksums": {},
"size": fileInfo["size"],
"events": fileInfo["events"],
"merged": 1,
"parents": fileInfo["parents"],
"locations": fileInfo["locations"],
"runs": acdcRuns
}
fileset.makeFilelist({lfn: acdcFile})
# Put the creation parameters
creationDict = jsonBlob["createRequest"]
creationDict["OriginalRequestName"] = str(workload.name())
creationDict["InitialTaskPath"] = requestObject['task']
creationDict["CollectionName"] = collectionName
creationDict["IgnoredOutputModules"] = list(
set(outputModules) - set(requestObject['outMod']))
creationDict["ACDCServer"] = acdcCouchUrl
creationDict["ACDCDatabase"] = acdcCouchDb
creationDict["RequestString"] = "recovery-%d-%s" % (
idx, workload.name()[:-18])
creationDict["Requestor"] = requestor
creationDict["Group"] = group
# Assign parameters
assignDict = jsonBlob["assignRequest"]
team = requestInfo['teams'][0]
processingString = workload.getProcessingString()
processingVersion = workload.getProcessingVersion()
acqEra = workload.getAcquisitionEra()
mergedLFNBase = workload.getMergedLFNBase()
unmergedLFNBase = workload.getUnmergedLFNBase()
topTask = workload.getTopLevelTask()[0]
siteWhitelist = topTask.siteWhitelist()
assignDict["SiteWhitelist"] = siteWhitelist
assignDict["MergedLFNBase"] = mergedLFNBase
assignDict["UnmergedLFNBase"] = unmergedLFNBase
assignDict["AcquisitionEra"] = acqEra
assignDict["Team"] = team
try:
int(processingVersion)
assignDict["ProcessingVersion"] = int(processingVersion)
if processingString is not None and processingString != 'None':
assignDict["ProcessingString"] = processingString
except Exception:
tokens = processingVersion.split('-')
assignDict["ProcessingVersion"] = int(tokens[-1][1:])
assignDict["ProcessingString"] = ('-').join(tokens[:-1])
fileHandle = open('%s.json' % creationDict["RequestString"], 'w')
json.dump(jsonBlob, fileHandle)
fileHandle.close()
logging.info("Created JSON %s for recovery of %s" %
('%s.json' % creationDict["RequestString"],
requestObject['outputs']))
logging.info("This will recover %d lumis in %d files" % (lumis, files))
def defineRequests(workload, requestInfo,
acdcCouchUrl, acdcCouchDb,
requestor, group,
dbsUrl,
fakeLocation,
datasetInformation = None):
"""
_defineRequests_
This is the ultimate function,
it will create JSONs for the appropiate resubmission requests
that can be feed into the reqmgr.py script and it will assemble
acdc records that can be uploaded to the database.
"""
# First retrieve the run and block lists and load
# the information of all datasets
logging.debug("Original request info:\n%s", requestInfo)
topTask = workload.getTopLevelTask()[0]
runWhitelist = topTask.inputRunWhitelist()
runBlacklist = topTask.inputRunBlacklist()
blockWhitelist = topTask.inputBlockWhitelist()
blockBlacklist = topTask.inputBlockBlacklist()
inputDataset = workload.listInputDatasets()[0]
outputModules = getOutputModules(workload)
if datasetInformation is None:
datasetInformation = {}
logging.info("Loading DBS information for the datasets...")
datasetInformation[inputDataset] = getFiles(inputDataset, runBlacklist, runWhitelist,
blockBlacklist, blockWhitelist, dbsUrl, fakeLocation=fakeLocation)
for dataset in workload.listOutputDatasets():
datasetInformation[dataset] = getFiles(dataset, runBlacklist, runWhitelist, blockBlacklist, blockWhitelist, dbsUrl)
logging.info("Finished loading DBS information for the datasets...")
# Now get the information about the datasets and tasks
nodes, edges = buildDatasetTree(workload)
logging.info("Dataset tree built...")
for k,v in nodes.items():
logging.debug("%s : %s" % (k,v))
for k,v in edges.items():
logging.debug("%s : %s" % (k,v))
# Load the difference information between input and outputs
differenceInformation = buildDifferenceMap(workload, datasetInformation)
logging.info("Difference map processed...")
logging.debug("%s" % str(differenceInformation))
# Define an object that will hold the potential requests
requests = []
logging.info("Now definining the required requests...")
# First generate requests for the datasets with children, that way we can
# shoot the requests with skims in single requests
for dataset in differenceInformation.keys():
if dataset not in nodes:
continue
datasetsToRecover = [dataset]
diffedLumis = differenceInformation[dataset]
taskToRecover = edges[(inputDataset, dataset)]['task']
outputModulesToRecover = edges[(inputDataset,dataset)]['outMod']
intersectionDiff = {}
for childDataset in nodes[dataset]:
childDiffLumis = differenceInformation[childDataset]
matchAvailable = False
for run in diffedLumis:
if run in childDiffLumis:
for lumi in diffedLumis[run]:
if lumi in childDiffLumis[run]:
matchAvailable = True
break
if matchAvailable:
outputModulesToRecover.extend(edges[(dataset, childDataset)]['outMod'])
datasetsToRecover.append(childDataset)
for run in diffedLumis:
if run in childDiffLumis:
if run not in intersectionDiff:
intersectionDiff[run] = set()
intersectionDiff[run] = diffedLumis[run] & childDiffLumis[run]
else:
intersectionDiff[run] &= diffedLumis[run] & childDiffLumis[run]
else:
intersectionDiff[run] = set()
for run in intersectionDiff:
if not intersectionDiff[run]:
del intersectionDiff[run]
if not intersectionDiff:
# Can't create request for this dataset + children
continue
for run in intersectionDiff:
for childDataset in nodes[dataset]:
childDiffLumis = differenceInformation[childDataset]
if run in childDiffLumis:
childDiffLumis[run] -= intersectionDiff[run]
if not childDiffLumis[run]:
del childDiffLumis[run]
diffedLumis[run] -= intersectionDiff[run]
if not diffedLumis[run]:
del diffedLumis[run]
if not diffedLumis:
del differenceInformation[dataset]
for childDataset in nodes[dataset]:
if not differenceInformation[childDataset]:
del differenceInformation[childDataset]
requestObject = {'task' : taskToRecover,
'input' : inputDataset,
'lumis' : intersectionDiff,
'outMod' : outputModulesToRecover,
'outputs' : datasetsToRecover}
requests.append(requestObject)
# Now go through all the output datasets, creating a single request for
# each
for dataset in differenceInformation:
datasetsToRecover = [dataset]
diffedLumis = differenceInformation[dataset]
if (inputDataset, dataset) in edges:
taskToRecover = edges[(inputDataset, dataset)]['task']
outputModulesToRecover = edges[(inputDataset,dataset)]['outMod']
parentDataset = inputDataset
else:
for parentDataset in nodes:
if dataset in nodes[parentDataset]:
taskToRecover = edges[(parentDataset, dataset)]['task']
outputModulesToRecover = edges[(parentDataset,dataset)]['outMod']
break
requestObject = {'task' : taskToRecover,
'input' : parentDataset,
'lumis' : diffedLumis,
'outMod' : outputModulesToRecover,
'outputs' : datasetsToRecover}
requests.append(requestObject)
logging.info("About to upload ACDC records to: %s/%s" % (acdcCouchUrl, acdcCouchDb))
pprint(requests)
# With the request objects we need to build ACDC records and
# request JSONs
for idx, requestObject in enumerate(requests):
collectionName = '%s_%s' % (workload.name(), str(uuid.uuid1()))
filesetName = requestObject['task']
collection = CouchCollection(**{"url" : acdcCouchUrl,
"database" : acdcCouchDb,
"name" : collectionName})
owner = makeUser(workload.getOwner()['group'], workload.getOwner()['name'], acdcCouchUrl, acdcCouchDb)
collection.setOwner(owner)
files = 0
lumis = 0
for lfn in datasetInformation[requestObject['input']]:
fileInfo = datasetInformation[requestObject['input']][lfn]
fileRuns = {}
for run in fileInfo['runs']:
if run in requestObject['lumis']:
for lumi in fileInfo['runs'][run][0]:
if lumi in requestObject['lumis'][run]:
if run not in fileRuns:
fileRuns[run] = []
fileRuns[run].append(lumi)
lumis += 1
if fileRuns:
files += 1
fileset = CouchFileset(**{"url" : acdcCouchUrl,
"database" : acdcCouchDb,
"name" : filesetName})
fileset.setCollection(collection)
acdcRuns = []
for run in fileRuns:
runObject = {}
runObject['run_number'] = int(run)
runObject['lumis'] = fileRuns[run]
acdcRuns.append(runObject)
acdcFile = {"lfn" : lfn,
"first_event" : 0,
"last_event" : 0,
"checksums" : {},
"size" : fileInfo["size"],
"events" : fileInfo["events"],
"merged" : 1,
"parents" : fileInfo["parents"],
"locations" : fileInfo["locations"],
"runs" : acdcRuns
}
fileset.makeFilelist({lfn : acdcFile})
# Put the creation parameters
creationDict = jsonBlob["createRequest"]
creationDict["OriginalRequestName"] = str(workload.name())
creationDict["InitialTaskPath"] = requestObject['task']
creationDict["CollectionName"] = collectionName
creationDict["IgnoredOutputModules"] = list(set(outputModules) - set(requestObject['outMod']))
creationDict["ACDCServer"] = acdcCouchUrl
creationDict["ACDCDatabase"] = acdcCouchDb
creationDict["RequestString"] = "recovery-%d-%s" % (idx, workload.name()[:-18])
creationDict["Requestor"] = requestor
creationDict["Group"] = group
creationDict["TimePerEvent"] = requestInfo['TimePerEvent']
creationDict["Memory"] = requestInfo['Memory']
creationDict["SizePerEvent"] = requestInfo['SizePerEvent']
creationDict["PrepID"] = requestInfo.get('PrepID')
creationDict["Campaign"] = requestInfo.get('Campaign')
# Assign parameters
assignDict = jsonBlob["assignRequest"]
team = requestInfo['Teams'][0]
processingString = requestInfo['ProcessingString']
processingVersion = requestInfo['ProcessingVersion']
acqEra = requestInfo['AcquisitionEra']
mergedLFNBase = requestInfo['MergedLFNBase']
unmergedLFNBase = requestInfo['UnmergedLFNBase']
#processingString = workload.getProcessingString()
#processingVersion = workload.getProcessingVersion()
#acqEra = workload.getAcquisitionEra()
#mergedLFNBase = workload.getMergedLFNBase()
#unmergedLFNBase = workload.getUnmergedLFNBase()
topTask = workload.getTopLevelTask()[0]
siteWhitelist = topTask.siteWhitelist()
assignDict["SiteWhitelist"] = siteWhitelist
assignDict["MergedLFNBase"] = mergedLFNBase
assignDict["UnmergedLFNBase"] = unmergedLFNBase
assignDict["AcquisitionEra"] = acqEra
assignDict["Team"] = team
try:
int(processingVersion)
assignDict["ProcessingVersion"] = int(processingVersion)
if processingString is not None and processingString != 'None':
assignDict["ProcessingString"] = processingString
except Exception:
tokens = processingVersion.split('-')
assignDict["ProcessingVersion"] = int(tokens[-1][1:])
assignDict["ProcessingString"] = ('-').join(tokens[:-1])
fileHandle = open('%s.json' % creationDict["RequestString"], 'w')
json.dump(jsonBlob, fileHandle)
fileHandle.close()
logging.info("Created JSON %s for recovery of %s" % ('%s.json' % creationDict["RequestString"],
requestObject['outputs']))
logging.info("This will recover %d lumis in %d files" % (lumis, files))
