def _createPileupConfigFile(self, helper):
"""
Stores pileup JSON configuration file in the working
directory / sandbox.
"""
encoder = JSONEncoder()
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
url = helper.data.dbsUrl
dbsReader = DBSReader(url)
configDict = self._queryDbsAndGetPileupConfig(helper, dbsReader)
# create JSON and save into a file
json = encoder.encode(configDict)
stepPath = "%s/%s" % (self.workingDirectory(), helper.name())
if not os.path.exists(stepPath):
os.mkdir(stepPath)
try:
fileName = "%s/%s" % (stepPath, "pileupconf.json")
f = open(fileName, 'w')
f.write(json)
f.close()
except IOError:
m = "Could not save pileup JSON configuration file: '%s'" % fileName
raise RuntimeError(m)
def _createPileupConfigFile(self, helper):
"""
Stores pileup JSON configuration file in the working
directory / sandbox.
"""
encoder = JSONEncoder()
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
url = helper.data.dbsUrl
dbsReader = DBSReader(url)
configDict = self._queryDbsAndGetPileupConfig(helper, dbsReader)
# create JSON and save into a file
json = encoder.encode(configDict)
stepPath = "%s/%s" % (self.workingDirectory(), helper.name())
if not os.path.exists(stepPath):
os.mkdir(stepPath)
try:
fileName = "%s/%s" % (stepPath, "pileupconf.json")
f = open(fileName, 'w')
f.write(json)
f.close()
except IOError:
m = "Could not save pileup JSON configuration file: '%s'" % fileName
raise RuntimeError(m)
def _createPileupConfigFile(self, helper, fakeSites=None):
"""
Stores pileup JSON configuration file in the working
directory / sandbox.
"""
if fakeSites is None:
fakeSites = []
if self._isCacheValid(helper):
# if file already exist don't make a new dbs call and overwrite the file.
# just return
return
encoder = JSONEncoder()
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
url = helper.data.dbsUrl
dbsReader = DBSReader(url)
configDict = self._queryDbsAndGetPileupConfig(helper, dbsReader,
fakeSites)
# create JSON and save into a file
json = encoder.encode(configDict)
self._saveFile(helper, json)
def _createPileupConfigFile(self, helper, fakeSites):
"""
Stores pileup JSON configuration file in the working
directory / sandbox.
"""
stepPath = "%s/%s" % (self.workingDirectory(), helper.name())
fileName = "%s/%s" % (stepPath, "pileupconf.json")
if os.path.isfile(fileName) and os.path.getsize(fileName) > 0:
# if file already exist don't make a new dbs call and overwrite the file.
# just return
return
encoder = JSONEncoder()
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
url = helper.data.dbsUrl
dbsReader = DBSReader(url)
configDict = self._queryDbsAndGetPileupConfig(helper, dbsReader, fakeSites)
# create JSON and save into a file
json = encoder.encode(configDict)
if not os.path.exists(stepPath):
os.mkdir(stepPath)
try:
fileName = "%s/%s" % (stepPath, "pileupconf.json")
f = open(fileName, 'w')
f.write(json)
f.close()
except IOError:
m = "Could not save pileup JSON configuration file: '%s'" % fileName
raise RuntimeError(m)
def _createPileupConfigFile(self, helper, fakeSites=None):
"""
Stores pileup JSON configuration file in the working
directory / sandbox.
"""
if fakeSites is None:
fakeSites = []
if self._isCacheValid(helper):
# we need to update the new sandbox json file in case TrustPUSitelists is on
if fakeSites:
self._updatePileupPNNs(helper, fakeSites)
# if file already exist don't make a new dbs call and overwrite the file.
# just return
return
encoder = JSONEncoder()
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
url = helper.data.dbsUrl
dbsReader = DBSReader(url)
configDict = self._queryDbsAndGetPileupConfig(helper, dbsReader, fakeSites)
# create JSON and save into a file
jsonPU = encoder.encode(configDict)
self._saveFile(helper, jsonPU)
def _createPileupConfigFile(self, helper):
"""
Stores pileup JSON configuration file in the working
directory / sandbox.
"""
encoder = JSONEncoder()
args = {}
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
args["url"] = helper.data.dbsUrl
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
from DBSAPI.dbsApi import DbsApi
dbsApi = DbsApi(args)
configDict = self._queryDbsAndGetPileupConfig(helper, dbsApi)
# create JSON and save into a file
json = encoder.encode(configDict)
stepPath = "%s/%s" % (self.workingDirectory(), helper.name())
if not os.path.exists(stepPath):
os.mkdir(stepPath)
try:
fileName = "%s/%s" % (stepPath, "pileupconf.json")
f = open(fileName, "w")
f.write(json)
f.close()
except IOError:
m = "Could not save pileup JSON configuration file: '%s'" % fileName
raise RuntimeError(m)
def encode(self, data):
"""
encode data as json
"""
encoder = JSONEncoder()
thunker = JSONThunker()
thunked = thunker.thunk(data)
return encoder.encode(thunked)
def encode(self, data):
"""
encode data as json
"""
encoder = JSONEncoder()
thunker = JSONThunker()
thunked = thunker.thunk(data)
return encoder.encode(thunked)
def updateRequestStatus(self, request, status):
statusTime = {'status': status, 'update_time': int(time.time())}
return self.couchDB.updateDocument(
request,
self.couchapp,
'requestStatus',
fields={'request_status': JSONEncoder().encode(statusTime)})
def insertGenericRequest(self, doc):
result = self.couchDB.updateDocument(
doc['_id'],
self.couchapp,
'insertRequest',
fields={'doc': JSONEncoder().encode(doc)})
self.updateRequestStatus(doc['_id'], "new")
return result
def _updatePileupPNNs(self, stepHelper, fakeSites):
"""
Update the workflow copy of the cached pileup file with PNNs
forced by TrustPUSitelists flag
"""
fileName = self._getStepFilePath(stepHelper)
fakePNNs = mapSitetoPNN(fakeSites)
with open(fileName, 'r') as puO:
pileupData = json.load(puO)
for dummyPUType, blockDict in pileupData.iteritems():
for dummyBlockName, blockInfo in blockDict.iteritems():
blockInfo['PhEDExNodeNames'].extend([x for x in fakePNNs if x not in blockInfo])
encoder = JSONEncoder()
jsonPU = encoder.encode(pileupData)
self._writeFile(fileName, jsonPU)
def updateRequestsInfo(self, docs):
"""
bulk update for request documents.
TODO: change to bulk update handler when it gets supported
"""
for doc in docs:
del doc['type']
self.couchDB.updateDocument(doc['workflow'], self.couchapp,
'generalFields',
fields={'general_fields': JSONEncoder().encode(doc)})
def updateRequestProperty(self, request, propDict):
encodeProperty = {}
for key, value in propDict.items():
if key in self._propertyNeedToBeEncoded:
encodeProperty[key] = JSONEncoder().encode(value)
else:
encodeProperty[key] = value
return self.couchDB.updateDocument(request, self.couchapp,
"updaterequest", encodeProperty)
def updateFromWMSpec(self, spec):
# currently only update priority and siteWhitelist and output dataset
# complex field needs to be JSON encoded
# assuming all the toplevel tasks has the same site white lists
#priority is priority + user priority + group priority
fields = {'priority': spec.priority(),
'site_white_list': spec.getTopLevelTask()[0].siteWhitelist(),
'outputdatasets': spec.listOutputDatasets()}
return self.couchDB.updateDocument(spec.name(), self.couchapp,
'generalFields',
fields={'general_fields': JSONEncoder().encode(fields)})
def updateRequestProperty(self, request, propDict, dn=None):
encodeProperty = {}
for key, value in propDict.items():
if isinstance(value, list) or isinstance(value, dict):
encodeProperty[key] = JSONEncoder().encode(value)
else:
encodeProperty[key] = value
if dn:
encodeProperty["DN"] = dn
return self.couchDB.updateDocument(request, self.couchapp,
"updaterequest", encodeProperty)
def _createPileupConfigFile(self, helper, fakeSites=None):
"""
Stores pileup JSON configuration file in the working
directory / sandbox.
"""
if fakeSites is None:
fakeSites = []
stepPath = "%s/%s" % (self.workingDirectory(), helper.name())
fileName = "%s/%s" % (stepPath, "pileupconf.json")
if os.path.isfile(fileName) and os.path.getsize(fileName) > 0:
# if file already exist don't make a new dbs call and overwrite the file.
# just return
return
encoder = JSONEncoder()
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
url = helper.data.dbsUrl
dbsReader = DBSReader(url)
configDict = self._queryDbsAndGetPileupConfig(helper, dbsReader, fakeSites)
# create JSON and save into a file
json = encoder.encode(configDict)
if not os.path.exists(stepPath):
os.mkdir(stepPath)
try:
fileName = "%s/%s" % (stepPath, "pileupconf.json")
f = open(fileName, 'w')
f.write(json)
f.close()
except IOError:
m = "Could not save pileup JSON configuration file: '%s'" % fileName
raise RuntimeError(m)
def replaceRequestTransitionFromReqMgr(self, docs):
"""
bulk update for request documents.
TODO: change to bulk update handler when it gets supported
"""
for doc in docs:
requestName = doc["RequestName"]
requestTransition = {}
requestTransition['request_status'] = []
for r in doc["RequestTransition"]:
newR = {}
newR['status'] = r['Status']
newR['update_time'] = r['UpdateTime']
requestTransition['request_status'].append(newR)
self.couchDB.updateDocument(requestName, self.couchapp,
'generalFields',
fields={'general_fields': JSONEncoder().encode(requestTransition)})
def updateAgentInfo(self, agentInfo):
return self.couchDB.updateDocument(
agentInfo['_id'],
'WMStats',
'agentInfo',
fields={'agent_info': JSONEncoder().encode(agentInfo)})
def updateAgentInfo(self, agentInfo):
return self.couchDB.updateDocument(
agentInfo['_id'],
self.couchapp,
'agentInfo', {'agent_info': JSONEncoder().encode(agentInfo)},
useBody=True)
