def checkCfgConfig(self, cfg_params):
"""
"""
self.datasvc_url = cfg_params.get("USER.datasvc_url", self.datasvc_url)
self.srm_version = cfg_params.get("USER.srm_version", 'srmv2')
self.node = cfg_params.get('USER.storage_element', None)
self.usenamespace = cfg_params.get("USER.usenamespace", 0)
self.user_remote_dir = cfg_params.get("USER.user_remote_dir", '')
self.publish_data = cfg_params.get("USER.publish_data", 0)
if int(self.publish_data) == 1:
# only accepts valid PhEDEx Node Names
import Lexicon
try:
Lexicon.cmsname(self.node)
except Exception, text:
msg = "%s\n'%s' is not a valid Phedex Node Name" % (text,
self.node)
msg += "\n***************** NOTICE ***************"
msg += "\nOnly valid Phexex Node Names can be used as location for published data"
msg += "\nPlease fix storage_element or set publish_data=0 in [USER] section of crab.cfg"
if 'group' in self.user_remote_dir:
msg += '\nIf you are trying to stage out to some /store/group area, you can do like:'
msg += '\n storage_element = T2_US_UCSD'
msg += '\n user_remote_dir = /store/group/foo/bar'
msg += "\n****************************************"
raise CrabException(msg)
def checkCfgConfig(self,cfg_params):
"""
"""
self.datasvc_url = cfg_params.get("USER.datasvc_url",self.datasvc_url)
self.srm_version = cfg_params.get("USER.srm_version",'srmv2')
self.node = cfg_params.get('USER.storage_element',None)
self.usenamespace = cfg_params.get("USER.usenamespace",0)
self.user_remote_dir = cfg_params.get("USER.user_remote_dir",'')
self.publish_data = cfg_params.get("USER.publish_data",0)
if int(self.publish_data) == 1:
# only accepts valid PhEDEx Node Names
import Lexicon
try:
Lexicon.cmsname(self.node)
except Exception, text:
msg = "%s\n'%s' is not a valid Phedex Node Name" % (text,self.node)
msg += "\n***************** NOTICE ***************"
msg += "\nOnly valid Phexex Node Names can be used as location for published data"
msg += "\nPlease fix storage_element or set publish_data=0 in [USER] section of crab.cfg"
if 'group' in self.user_remote_dir:
msg += '\nIf you are trying to stage out to some /store/group area, you can do like:'
msg += '\n storage_element = T2_US_UCSD'
msg += '\n user_remote_dir = /store/group/foo/bar'
msg += "\n****************************************"
raise CrabException(msg)
def expandIntoListOfPhedexNodeNames(location_list):
"""
take as input a list of locations, returns a list of PNN's
raise CrabExceptoin if input is not a valid PNN abbreviation
use https://cmsweb.cern.ch/phedex/datasvc/doc/nodes
"""
# build API node filter, add wildcards wich are not required by Crab2
args = ''
for loc in location_list:
phedexNode = loc.strip()
try:
Lexicon.cmsname(phedexNode)
except Exception, text:
msg = "%s\n'%s' is not a valid Phedex Node Name" % (text,phedexNode)
raise CrabException(msg)
args += "&node=%s*" % phedexNode
def expandIntoListOfPhedexNodeNames(location_list):
"""
take as input a list of locations, returns a list of PNN's
raise CrabExceptoin if input is not a valid PNN abbreviation
use https://cmsweb.cern.ch/phedex/datasvc/doc/nodes
"""
# build API node filter, add wildcards wich are not required by Crab2
args = ''
for loc in location_list:
phedexNode = loc.strip()
try:
Lexicon.cmsname(phedexNode)
except Exception, text:
msg = "%s\n'%s' is not a valid Phedex Node Name" % (text,
phedexNode)
raise CrabException(msg)
args += "&node=%s*" % phedexNode
def validateBWLists(cfg_params):
# convert to lists for processing. But leave cfg_params
# as strings, since this is what Crab2 code expects
blackList = cfg_params.get("GRID.se_black_list", [])
if type(blackList) == type("string"):
blackList = blackList.strip().split(',')
whiteList = cfg_params.get("GRID.se_white_list", [])
if type(whiteList) == type("string"):
whiteList = whiteList.strip().split(',')
# make sure each item in the list is a valid cms node name
# or possibly a shortcut like T3
for site in blackList:
try:
Lexicon.cmsname(site)
except Exception, text:
msg = "ERROR in GRID.se_black_list: %s\n" % blackList
msg += "%s\n'%s' is not a valid Phedex Node Name" % (text, site)
raise CrabException(msg)
def validateBWLists(cfg_params):
# convert to lists for processing. But leave cfg_params
# as strings, since this is what Crab2 code expects
blackList = cfg_params.get("GRID.se_black_list", [] )
if type(blackList) == type("string") :
blackList = blackList.strip().split(',')
whiteList = cfg_params.get("GRID.se_white_list", [] )
if type(whiteList) == type("string") :
whiteList = whiteList.strip().split(',')
# make sure each item in the list is a valid cms node name
# or possibly a shortcut like T3
for site in blackList:
try:
Lexicon.cmsname(site)
except Exception, text:
msg = "ERROR in GRID.se_black_list: %s\n" % blackList
msg += "%s\n'%s' is not a valid Phedex Node Name" % (text,site)
raise CrabException(msg)
whiteList = whiteList.strip().split(',')
# make sure each item in the list is a valid cms node name
# or possibly a shortcut like T3
for site in blackList:
try:
Lexicon.cmsname(site)
except Exception, text:
msg = "ERROR in GRID.se_black_list: %s\n" % blackList
msg += "%s\n'%s' is not a valid Phedex Node Name" % (text, site)
raise CrabException(msg)
for site in whiteList:
try:
Lexicon.cmsname(site)
except Exception, text:
msg = "ERROR in GRID.se_white_list: %s\n" % whiteList
msg += "%s\n'%s' is not a valid Phedex Node Name" % (text, site)
raise CrabException(msg)
def parseIntoList(param):
"""
to be used to make sure that one crab config parameter is usable as a list of strings,
eve if it is a string with comma insides in the config. file
"""
if type(param) == type("string"):
list = param.split(',')
for item in list:
item = item.strip()
whiteList = whiteList.strip().split(',')
# make sure each item in the list is a valid cms node name
# or possibly a shortcut like T3
for site in blackList:
try:
Lexicon.cmsname(site)
except Exception, text:
msg = "ERROR in GRID.se_black_list: %s\n" % blackList
msg += "%s\n'%s' is not a valid Phedex Node Name" % (text,site)
raise CrabException(msg)
for site in whiteList:
try:
Lexicon.cmsname(site)
except Exception, text:
msg = "ERROR in GRID.se_white_list: %s\n" % whiteList
msg += "%s\n'%s' is not a valid Phedex Node Name" % (text,site)
raise CrabException(msg)
def parseIntoList(param):
"""
to be used to make sure that one crab config parameter is usable as a list of strings,
eve if it is a string with comma insides in the config. file
"""
if type(param) == type("string") :
list = param.split(',')
for item in list:
item = item.strip()
writeUrl = url + 'DBSWriter'
readApi = DbsApi(url=readUrl)
writeApi = DbsApi(url=writeUrl)
dataset = options.dataset
if options.new_location:
new_location = options.new_location
###sanitize input
# dataset name
Lexicon.dataset(dataset)
# PNN
if new_location:
Lexicon.cmsname(new_location)
# process dataset by blocks
blockDicts = readApi.listBlocks(dataset=dataset, detail=True)
for block in blockDicts:
blName = block['block_name']
location = block['origin_site_name']
logging.debug('block %s at location: %s' % (blName, location))
if new_location:
writeApi.updateBlockSiteName(block_name=blName, origin_site_name=new_location)
logging.debug('location set to %s' % (new_location))
logging.info("Done")
writeUrl = url + 'DBSWriter'
readApi = DbsApi(url=readUrl)
writeApi = DbsApi(url=writeUrl)
dataset = options.dataset
if options.new_location:
new_location = options.new_location
###sanitize input
# dataset name
Lexicon.dataset(dataset)
# PNN
if new_location:
Lexicon.cmsname(new_location)
# process dataset by blocks
blockDicts = readApi.listBlocks(dataset=dataset, detail=True)
for block in blockDicts:
blName = block['block_name']
location = block['origin_site_name']
logging.debug('block %s at location: %s' % (blName, location))
if new_location:
writeApi.updateBlockSiteName(block_name=blName,
origin_site_name=new_location)
logging.debug('location set to %s' % (new_location))
logging.info("Done")