def listFileBlockLocation(self, fileBlockName, dbsOnly = False, phedexNodes=False):
"""
_listFileBlockLocation_
Get origin_site_name of a block
"""
blockNames = [fileBlockName] if isinstance(fileBlockName, basestring) else fileBlockName
for block in blockNames:
self.checkBlockName(block)
blockInfo = {}
if not dbsOnly:
try:
blockInfo = self.phedex.getReplicaSEForBlocks(phedexNodes=phedexNodes, block=blockNames, complete='y')
except Exception as ex:
msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockName
msg += "%s\n" % str(ex)
raise Exception(msg)
if not blockInfo or len(blockInfo) != len(blockNames): #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
dbsOnly = True
blockNames = set(blockNames) - set(blockInfo) #get the blocks we did not find information in phedex
if dbsOnly:
try:
for block in blockNames:
res = self.dbs.listBlockOrigin(block_name = block)
if res:
blockInfo[block] = [res[0]['origin_site_name']]
except dbsClientException as ex:
msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not any(blockInfo.values()): # no data location from dbs
return list()
#removing duplicates and 'UNKNOWN entries
locations = {}
node_filter_list = set(['UNKNOWN', None])
for name, nodes in blockInfo.iteritems():
final_nodes = set()
for n in nodes:
if n in node_filter_list:
continue
try:
cmsname(n)
except AssertionError: ## is SE
n = self.phedex.getNodeNames(n) if phedexNodes else [n]
else: ## not SE i.e. phedexNode
n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
final_nodes = final_nodes.union(n)
locations[name] = list(final_nodes - node_filter_list)
#returning single list if a single block is passed
if isinstance(fileBlockName, basestring):
locations = locations[fileBlockName]
return locations
def validate_name(self, input):
if 'name' not in input.keys():
if input['scheme'] == 'cms_name':
input['name'] ='T%'
else:
input['name'] = 'CERN'
if input['scheme'] == 'cms_name' and input['name'] !='%':
for name in self.makelist(input['name']):
cmsname(name)
elif input['scheme'] == 'resource':
reg = "(\/|\/([\w#!:.?+=&%@!\-\/]))?"
assert re.compile(reg).match(input['name']) != None , \
"'%s' is not a valid FQDN (regexp: %s)" % (input['name'], reg)
return input
def validate_name(self, input):
if 'name' not in input.keys():
if input['scheme'] == 'cms_name':
input['name'] = 'T%'
else:
input['name'] = 'CERN'
if input['scheme'] == 'cms_name' and input['name'] != '%':
for name in self.makelist(input['name']):
cmsname(name)
elif input['scheme'] == 'resource':
reg = "(\/|\/([\w#!:.?+=&%@!\-\/]))?"
assert re.compile(reg).match(input['name']) != None , \
"'%s' is not a valid FQDN (regexp: %s)" % (input['name'], reg)
return input
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
"InputDataset": {"optional": False, "validate": dataset, "null": False},
"ConfigCacheID": {"optional": True, "null": True},
"DataTier": {"default": "USER", "type": str,
"optional": True, "validate": None,
"attr": "dataTier", "null": False},
"PhysicsGroup": {"default": "", "optional": False,
"null": False, "validate": physicsgroup},
"MergedLFNBase": {"default": "/store/results", "type": str,
"optional": True, "validate": None,
"attr": "mergedLFNBase", "null": False},
# site whitelist shouldn't be allowed, but let's make an exception for StoreResults
"SiteWhitelist": {"default": [], "type": makeNonEmptyList, "assign_optional": False,
"validate": lambda x: all([cmsname(y) for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist", "null": False},
"BlockWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist", "null": False},
"RunBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist", "null": False},
"RunWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist", "null": False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def do_options(defaults):
"""
Read the users arguments and set sensible defaults
"""
# Configure the options from the passed in list
op = OptionParser()
for key in defaults:
# Deep copy so we don't overwrite the default
cur = copy.deepcopy(defaults[key])
if cur['persist'] == True:
cur['opts']['default'] = None
op.add_option(key, cur['long'], **cur['opts'])
# Run the option parser
options, args = op.parse_args()
# Configure the logger
logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger('StageManager')
if options.verbose:
logger.setLevel(logging.INFO)
if options.debug:
logger.setLevel(logging.DEBUG)
# Sanity checks
logger.info('options: %s, args: %s' % (options, args))
if not cmsname(options.site):
logger.warning('%s is not a valid CMS name!' % options.site)
sys.exit(101)
elif not options.site.startswith('T1'):
logger.warning('%s is not a T1 site' % options.site)
sys.exit(102)
return options, args, logger
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
"InputDataset": {"optional": False, "validate": dataset, "null": False},
"ConfigCacheID": {"optional": True, "null": True},
"DataTier": {"default": "USER", "type": str,
"optional": True, "validate": None,
"attr": "dataTier", "null": False},
"PhysicsGroup": {"default": "", "optional": False,
"null": False, "validate": physicsgroup},
"MergedLFNBase": {"default": "/store/results", "type": str,
"optional": True, "validate": None,
"attr": "mergedLFNBase", "null": False},
# site whitelist shouldn't be allowed, but let's make an exception for StoreResults
"SiteWhitelist": {"default": [], "type": makeNonEmptyList, "assign_optional": False,
"validate": lambda x: all([cmsname(y) for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist", "null": False},
"BlockWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist", "null": False},
"RunBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist", "null": False},
"RunWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist", "null": False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
specArgs = {
"RequestType": {
"default": "Resubmission"
},
"ResubmissionCount": {
"default": 1,
"type": int
},
"OriginalRequestType": {
"null": False
},
"OriginalRequestName": {
"null": False
},
"InitialTaskPath": {
"optional": False,
"validate": lambda x: len(x.split('/')) > 2
},
"ACDCServer": {
"default": "https://cmsweb.cern.ch/couchdb",
"validate": couchurl,
"attr": "acdcServer"
},
"ACDCDatabase": {
"default": "acdcserver",
"validate": identifier,
"attr": "acdcDatabase"
},
"CollectionName": {
"default": None,
"null": True
},
"IgnoredOutputModules": {
"default": [],
"type": makeList
},
"SiteWhitelist": {
"default": [],
"type": makeList,
"validate": lambda x: all([cmsname(y) for y in x])
},
# it can be Chained or MC requests, so lets make it optional
"InputDataset": {
"optional": True,
"validate": dataset,
"null": True
}
}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getWorkloadCreateArgs():
specArgs = {"RequestType": {"default": "Resubmission"},
"OriginalRequestType": {"null": False},
"OriginalRequestName": {"null": False},
"InitialTaskPath": {"optional": False,
"validate": lambda x: len(x.split('/')) > 2},
"ACDCServer": {"default": "https://cmsweb.cern.ch/couchdb", "validate": couchurl,
"attr": "acdcServer"},
"ACDCDatabase": {"default": "acdcserver", "validate": identifier,
"attr": "acdcDatabase"},
"CollectionName": {"default": None, "null": True},
"IgnoredOutputModules": {"default": [], "type": makeList},
"SiteWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([cmsname(y) for y in x])},
# it can be Chained or MC requests, so lets make it optional
"InputDataset": {"optional": True, "validate": dataset, "null": True}}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getWorkloadArguments():
"""
_getWorkloadArguments_
This represents the authorative list of request arguments that are
interpreted by the current spec class.
The list is formatted as a 2-level dictionary, the keys in the first level
are the identifiers for the arguments processed by the current spec.
The second level dictionary contains the information about that argument for
validation:
- default: Gives a default value if not provided,
this default value usually is good enough for a standard workflow. If the argument is not optional
and a default value is provided, this is only meant for test purposes.
- type: A function that verifies the type of the argument, it may also cast it into the appropiate python type.
If the input is not compatible with the expected type, this method must throw an exception.
- optional: This boolean value indicates if the value must be provided or not
- validate: A function which validates the input after type casting,
it returns True if the input is valid, it can throw exceptions on invalid input.
- attr: This represents the name of the attribute corresponding to the argument in the WMSpec object.
- null: This indicates if the argument can have None as its value.
Example:
{
RequestPriority : {'default' : 0,
'type' : int,
'optional' : False,
'validate' : lambda x : x > 0,
'attr' : 'priority',
'null' : False}
}
This replaces the old syntax in the __call__ of:
self.priority = arguments.get("RequestPriority", 0)
"""
arguments = {"RequestPriority": {"default" : 0, "type" : int,
"optional" : False, "validate" : lambda x : (x >= 0 and x < 1e6),
"attr" : "priority"},
"Requestor": {"default" : "unknown", "optional" : False,
"attr" : "owner"},
"RequestorDN" : {"default" : "unknown", "optional" : False,
"attr" : "owner_dn"},
"Group" : {"default" : "unknown", "optional" : False,
"attr" : "group"},
"VoGroup" : {"default" : "DEFAULT", "attr" : "owner_vogroup"},
"VoRole" : {"default" : "DEFAULT", "attr" : "owner_vorole"},
"AcquisitionEra" : {"default" : "None", "attr" : "acquisitionEra",
"validate" : acqname},
"CMSSWVersion" : {"default" : "CMSSW_5_3_7", "validate" : cmsswversion,
"optional" : False, "attr" : "frameworkVersion"},
"ScramArch" : {"default" : "slc5_amd64_gcc462", "optional" : False},
"ProcessingVersion" : {"default" : 0, "attr" : "processingVersion",
"type" : int},
"ProcessingString" : {"default" : None, "attr" : "processingString",
"null" : True},
"SiteBlacklist" : {"default" : [], "type" : makeList,
"validate" : lambda x: all([cmsname(y) for y in x])},
"SiteWhitelist" : {"default" : [], "type" : makeList,
"validate" : lambda x: all([cmsname(y) for y in x])},
"UnmergedLFNBase" : {"default" : "/store/unmerged"},
"MergedLFNBase" : {"default" : "/store/data"},
"MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
"validate" : lambda x : x > 0},
"MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
"validate" : lambda x : x > 0},
"MaxWaitTime" : {"default" : 24 * 3600, "type" : int,
"validate" : lambda x : x > 0},
"MaxMergeEvents" : {"default" : 100000, "type" : int,
"validate" : lambda x : x > 0},
"ValidStatus" : {"default" : "PRODUCTION"},
"DbsUrl" : {"default" : "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"},
"DashboardHost" : {"default" : "cms-wmagent-job.cern.ch"},
"DashboardPort" : {"default" : 8884, "type" : int,
"validate" : lambda x : x > 0},
"OverrideCatalog" : {"default" : None, "null" : True},
"RunNumber" : {"default" : 0, "type" : int},
"TimePerEvent" : {"default" : 12.0, "type" : float,
"optional" : False, "validate" : lambda x : x > 0},
"Memory" : {"default" : 2300.0, "type" : float,
"optional" : False, "validate" : lambda x : x > 0},
"SizePerEvent" : {"default" : 512.0, "type" : float,
"optional" : False, "validate" : lambda x : x > 0},
"PeriodicHarvestInterval" : {"default" : 0, "type" : int,
"validate" : lambda x : x >= 0},
"DQMUploadProxy" : {"default" : None, "null" : True,
"attr" : "dqmUploadProxy"},
"DQMUploadUrl" : {"default" : "https://cmsweb.cern.ch/dqm/dev",
"attr" : "dqmUploadUrl"},
"DQMSequences" : {"default" : [], "type" : makeList,
"attr" : "dqmSequences"},
"DQMConfigCacheID" : {"default" : None, "null" : True,
"attr" : "dqmConfigCacheID"},
"EnableHarvesting" : {"default" : False, "type" : strToBool},
"EnableNewStageout" : {"default" : False, "type" : strToBool},
"IncludeParents" : {"default" : False, "type" : strToBool},
"Multicore" : {"default" : None, "null" : True,
"validate" : lambda x : x == "auto" or (int(x) > 0)}}
# Set defaults for the argument specification
for arg in arguments:
arguments[arg].setdefault("type", str)
arguments[arg].setdefault("optional", True)
arguments[arg].setdefault("null", False)
arguments[arg].setdefault("validate", None)
arguments[arg].setdefault("attr", arg[:1].lower() + arg[1:])
return arguments
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not any(blockInfo.values()): # no data location from dbs
return list()
#removing duplicates and 'UNKNOWN entries
locations = {}
node_filter_list = set(['UNKNOWN', None])
for name, nodes in blockInfo.iteritems():
final_nodes = set()
for n in nodes:
if n in node_filter_list:
continue
try:
cmsname(n)
except AssertionError: ## is SE
n = self.phedex.getNodeNames(n) if phedexNodes else [n]
else: ## not SE i.e. phedexNode
n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
final_nodes = final_nodes.union(n)
locations[name] = list(final_nodes - node_filter_list)
#returning single list if a single block is passed
if isinstance(fileBlockName, basestring):
locations = locations[fileBlockName]
return locations
def getFileBlock(self, fileBlockName):
"""
def getWorkloadArguments():
"""
_getWorkloadArguments_
This represents the authorative list of request arguments that are
interpreted by the current spec class.
The list is formatted as a 2-level dictionary, the keys in the first level
are the identifiers for the arguments processed by the current spec.
The second level dictionary contains the information about that argument for
validation:
- default: Gives a default value if not provided,
this default value usually is good enough for a standard workflow. If the argument is not optional
and a default value is provided, this is only meant for test purposes.
- type: A function that verifies the type of the argument, it may also cast it into the appropiate python type.
If the input is not compatible with the expected type, this method must throw an exception.
- optional: This boolean value indicates if the value must be provided or not by user
or inherited class can overwrite with default value.
- assign_optional: This boolean value indicates if the value must be provided when workflow is assinged if False.
- validate: A function which validates the input after type casting,
it returns True if the input is valid, it can throw exceptions on invalid input.
- attr: This represents the name of the attribute corresponding to the argument in the WMSpec object.
- null: This indicates if the argument can have None as its value.
If above is not specifyed, automatically set by following default value
- default: None
- type: str
- optional: True
- assign_optional: True
- validate: None
- attr: change first letter to lower case
- null: False
Example:
{
RequestPriority : {'default' : 0,
'type' : int,
'optional' : False,
'validate' : lambda x : x > 0,
'attr' : 'priority',
'null' : False}
}
This replaces the old syntax in the __call__ of:
self.priority = arguments.get("RequestPriority", 0)
"""
# if key is not specified it is set by default value
arguments = {"RequestType" : {"optional" : False}, # this need to be overwritten by inherited class
"Requestor": {"default": "unknown", "attr" : "owner"},
"RequestorDN" : {"default": "unknown", "attr" : "owner_dn"},
"Group" : {"default": "unknown"},
"RequestPriority": {"default" : 8000, "type" : int,
"validate" : lambda x : (x >= 0 and x < 1e6),
"attr" : "priority"},
"VoGroup" : {"default" : "unknown", "attr" : "owner_vogroup"},
"VoRole" : {"default" : "unknown", "attr" : "owner_vorole"},
"Campaign" : {"default" : ""},
"AcquisitionEra" : {"default" : "FAKE", "validate" : acqname, "assign_optional": False},
"CMSSWVersion" : {"validate" : cmsswversion,
"optional" : False, "attr" : "frameworkVersion"},
"ScramArch" : {"default" : "slc5_amd64_gcc462", "optional" : False},
"GlobalTag" : {"null" : True},
"GlobalTagConnect" : {"null" : True},
"ProcessingVersion" : {"default" : 1, "type" : int},
"ProcessingString" : {"null" : True},
"LumiList" : {"default" : {}, "type" : makeLumiList},
"SiteBlacklist" : {"default" : [], "type" : makeList,
"validate" : lambda x: all([cmsname(y) for y in x])},
"SiteWhitelist" : {"default" : [], "type" : makeList,
"validate" : lambda x: all([cmsname(y) for y in x])},
"TrustSitelists" : {"default" : False, "type" : strToBool},
"UnmergedLFNBase" : {"default" : "/store/unmerged"},
"MergedLFNBase" : {"default" : "/store/data"},
"MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
"validate" : lambda x : x > 0},
"MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
"validate" : lambda x : x > 0},
"MaxWaitTime" : {"default" : 24 * 3600, "type" : int,
"validate" : lambda x : x > 0},
"MaxMergeEvents" : {"default" : 100000, "type" : int,
"validate" : lambda x : x > 0},
"ValidStatus" : {"default" : "PRODUCTION"},
"DbsUrl" : {"default" : "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
"null" : True, "validate" : checkDBSURL},
"DashboardHost" : {"default" : "cms-wmagent-job.cern.ch"},
"DashboardPort" : {"default" : 8884, "type" : int,
"validate" : lambda x : x > 0},
"OverrideCatalog" : {"null" : True},
"RunNumber" : {"default" : 0, "type" : int},
"TimePerEvent" : {"default" : 12.0, "type" : float,
"validate" : lambda x : x > 0},
"Memory" : {"default" : 2300.0, "type" : float,
"validate" : lambda x : x > 0},
"SizePerEvent" : {"default" : 512.0, "type" : float,
"validate" : lambda x : x > 0},
"PeriodicHarvestInterval" : {"default" : 0, "type" : int,
"validate" : lambda x : x >= 0},
"DQMHarvestUnit" : {"default" : "byRun", "type" : str, "attr" : "dqmHarvestUnit"},
"DQMUploadProxy" : {"null" : True, "attr" : "dqmUploadProxy"},
"DQMUploadUrl" : {"default" : "https://cmsweb.cern.ch/dqm/dev", "attr" : "dqmUploadUrl"},
"DQMSequences" : {"default" : [], "type" : makeList, "attr" : "dqmSequences"},
"DQMConfigCacheID" : {"null" : True, "attr" : "dqmConfigCacheID"},
"EnableHarvesting" : {"default" : False, "type" : strToBool},
"EnableNewStageout" : {"default" : False, "type" : strToBool},
"IncludeParents" : {"default" : False, "type" : strToBool},
"Multicore" : {"default" : 1, "null" : True,
"validate" : lambda x : x == "auto" or (int(x) > 0)},
#from assignment: performance monitoring data
"MaxRSS" : {"default" : 2411724, "type" : int, "validate" : lambda x : x > 0},
"MaxVSize" : {"default" : 20411724, "type" : int, "validate" : lambda x : x > 0},
"SoftTimeout" : {"default" : 129600, "type" : int, "validate" : lambda x : x > 0},
"GracePeriod" : {"default" : 300, "type" : int, "validate" : lambda x : x > 0},
"UseSiteListAsLocation" : {"default" : False, "type" : bool},
# Set phedex subscription information
"CustodialSites" : {"default" : [], "type" : makeList, "assign_optional": True,
"validate" : lambda x: all([cmsname(y) for y in x])},
"NonCustodialSites" : {"default" : [], "type" : makeList, "assign_optional": True,
"validate" : lambda x: all([cmsname(y) for y in x])},
"AutoApproveSubscriptionSites" : {"default" : [], "type" : makeList, "assign_optional": True,
"validate" : lambda x: all([cmsname(y) for y in x])},
# should be Low, Normal, High
"SubscriptionPriority" : {"default" : "Low", "assign_optional": True,
"validate" : lambda x: x in ["Low", "Normal", "High"]},
# should be Move Replica
"CustodialSubType" : {"default" : "Replica", "type" : str, "assign_optional": True,
"validate" : lambda x: x in ["Move", "Replica"]},
"NonCustodialSubType" : {"default" : "Replica", "type" : str, "assign_optional": True,
"validate" : lambda x: x in ["Move", "Replica"]},
# Block closing informaiont
"BlockCloseMaxWaitTime" : {"default" : 66400, "type" : int, "validate" : lambda x : x > 0},
"BlockCloseMaxFiles" : {"default" : 500, "type" : int, "validate" : lambda x : x > 0},
"BlockCloseMaxEvents" : {"default" : 25000000, "type" : int, "validate" : lambda x : x > 0},
"BlockCloseMaxSize" : {"default" : 5000000000000, "type" : int, "validate" : lambda x : x > 0},
# dashboard activity
"Dashboard" : {"default": "", "type" : str},
# team name
"Team" : {"default": "", "type" : str},
"PrepID": {"default" : None, "null" : True}
}
# Set defaults for the argument specification
StdBase.setDefaultArgumentsProperty(arguments)
return arguments
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not any(blockInfo.values()): # no data location from dbs
return list()
#removing duplicates and 'UNKNOWN entries
locations = {}
node_filter_list = set(['UNKNOWN', None])
for name, nodes in blockInfo.iteritems():
final_nodes = set()
for n in nodes:
if n in node_filter_list:
continue
try:
cmsname(n)
except AssertionError: ## is SE
n = self.phedex.getNodeNames(n) if phedexNodes else [n]
else: ## not SE i.e. phedexNode
n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
final_nodes = final_nodes.union(n)
locations[name] = list(final_nodes - node_filter_list)
#returning single list if a single block is passed
if isinstance(fileBlockName, basestring):
locations = locations[fileBlockName]
return locations
def getFileBlock(self, fileBlockName):
"""
def do_options():
op = OptionParser()
op.add_option("-u", "--url",
type="string",
action="store",
dest="couch",
help="CouchDB url. Default address http://127.0.0.1:5985",
default="http://127.0.0.1:5985")
op.add_option("-s", "--site",
dest="site",
default="",
help="The T1 site you want to query")
op.add_option("-n", "--new",
dest="new",
action="store_true",
default=False,
help="Show status of new requests")
op.add_option("-a", "--acquired",
dest="acquired",
action="store_true",
default=False,
help="Show status of acquired requests")
op.add_option("-d", "--done",
dest="done",
action="store_true",
default=False,
help="Show status of done requests")
op.add_option("-e", "--expired",
dest="expired",
action="store_true",
default=False,
help="Show status of expired requests")
op.add_option("-q", "--no-detail",
dest="nodetail",
action="store_true",
default=False,
help="Hide the pre-requests information and show only totals")
op.add_option("-v", "--data",
dest="data",
action="store_true",
default=False,
help="Show the data requested against each status")
options, args = op.parse_args()
logger = logging.getLogger('StageManager')
if options.site != "":
if not cmsname(options.site):
logger.critical('%s is not a valid CMS name!' % options.site)
sys.exit(101)
if not options.site.startswith('T1'):
logger.critical('%s is not a T1 site' % options.site)
sys.exit(102)
else:
logger.critical('you need to provide a T1 site to stage at (-s option)')
sys.exit(103)
return options, args, logger
def getWorkloadArguments():
"""
_getWorkloadArguments_
This represents the authorative list of request arguments that are
interpreted by the current spec class.
The list is formatted as a 2-level dictionary, the keys in the first level
are the identifiers for the arguments processed by the current spec.
The second level dictionary contains the information about that argument for
validation:
- default: Gives a default value if not provided,
this default value usually is good enough for a standard workflow. If the argument is not optional
and a default value is provided, this is only meant for test purposes.
- type: A function that verifies the type of the argument, it may also cast it into the appropiate python type.
If the input is not compatible with the expected type, this method must throw an exception.
- optional: This boolean value indicates if the value must be provided or not
- validate: A function which validates the input after type casting,
it returns True if the input is valid, it can throw exceptions on invalid input.
- attr: This represents the name of the attribute corresponding to the argument in the WMSpec object.
- null: This indicates if the argument can have None as its value.
Example:
{
RequestPriority : {'default' : 0,
'type' : int,
'optional' : False,
'validate' : lambda x : x > 0,
'attr' : 'priority',
'null' : False}
}
This replaces the old syntax in the __call__ of:
self.priority = arguments.get("RequestPriority", 0)
"""
arguments = {"RequestPriority": {"default" : 0, "type" : int,
"optional" : False, "validate" : lambda x : (x >= 0 and x < 1e6),
"attr" : "priority"},
"Requestor": {"default" : "unknown", "optional" : False,
"attr" : "owner"},
"RequestorDN" : {"default" : "unknown", "optional" : False,
"attr" : "owner_dn"},
"Group" : {"default" : "unknown", "optional" : False,
"attr" : "group"},
"VoGroup" : {"default" : "DEFAULT", "attr" : "owner_vogroup"},
"VoRole" : {"default" : "DEFAULT", "attr" : "owner_vorole"},
"AcquisitionEra" : {"default" : "None", "attr" : "acquisitionEra",
"validate" : acqname},
"CMSSWVersion" : {"default" : "CMSSW_5_3_7", "validate" : cmsswversion,
"optional" : False, "attr" : "frameworkVersion"},
"ScramArch" : {"default" : "slc5_amd64_gcc462", "optional" : False},
"GlobalTag" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "globalTag", "null" : True},
"GlobalTagConnect" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "globalTagConnect", "null" : True},
"ProcessingVersion" : {"default" : 0, "attr" : "processingVersion",
"type" : int},
"ProcessingString" : {"default" : None, "attr" : "processingString",
"null" : True},
"LumiList" : {"default" : [], "type" : makeLumiList,
"optional" : True, "validate" : None,
"attr" : "lumiList", "null" : False},
"SiteBlacklist" : {"default" : [], "type" : makeList,
"validate" : lambda x: all([cmsname(y) for y in x])},
"SiteWhitelist" : {"default" : [], "type" : makeList,
"validate" : lambda x: all([cmsname(y) for y in x])},
"UnmergedLFNBase" : {"default" : "/store/unmerged"},
"MergedLFNBase" : {"default" : "/store/data"},
"MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
"validate" : lambda x : x > 0},
"MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
"validate" : lambda x : x > 0},
"MaxWaitTime" : {"default" : 24 * 3600, "type" : int,
"validate" : lambda x : x > 0},
"MaxMergeEvents" : {"default" : 100000, "type" : int,
"validate" : lambda x : x > 0},
"ValidStatus" : {"default" : "PRODUCTION"},
"DbsUrl" : {"default" : "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
"null" : True, "validate" : checkDBSUrl},
"DashboardHost" : {"default" : "cms-wmagent-job.cern.ch"},
"DashboardPort" : {"default" : 8884, "type" : int,
"validate" : lambda x : x > 0},
"OverrideCatalog" : {"default" : None, "null" : True},
"RunNumber" : {"default" : 0, "type" : int},
"TimePerEvent" : {"default" : 12.0, "type" : float,
"optional" : False, "validate" : lambda x : x > 0},
"Memory" : {"default" : 2300.0, "type" : float,
"optional" : False, "validate" : lambda x : x > 0},
"SizePerEvent" : {"default" : 512.0, "type" : float,
"optional" : False, "validate" : lambda x : x > 0},
"PeriodicHarvestInterval" : {"default" : 0, "type" : int,
"validate" : lambda x : x >= 0},
"DQMUploadProxy" : {"default" : None, "null" : True,
"attr" : "dqmUploadProxy"},
"DQMUploadUrl" : {"default" : "https://cmsweb.cern.ch/dqm/dev",
"attr" : "dqmUploadUrl"},
"DQMSequences" : {"default" : [], "type" : makeList,
"attr" : "dqmSequences"},
"DQMConfigCacheID" : {"default" : None, "null" : True,
"attr" : "dqmConfigCacheID"},
"EnableHarvesting" : {"default" : False, "type" : strToBool},
"EnableNewStageout" : {"default" : False, "type" : strToBool},
"IncludeParents" : {"default" : False, "type" : strToBool},
"Multicore" : {"default" : None, "null" : True,
"validate" : lambda x : x == "auto" or (int(x) > 0)},
"PrepID": {"default" : None, "null" : True}}
# Set defaults for the argument specification
StdBase.setDefaultArgumentsProperty(arguments)
return arguments
def listFileBlockLocation(self,
fileBlockName,
dbsOnly=False,
phedexNodes=False):
"""
_listFileBlockLocation_
Get origin_site_name of a block
"""
blockNames = [fileBlockName] if isinstance(
fileBlockName, basestring) else fileBlockName
for block in blockNames:
self.checkBlockName(block)
blockInfo = {}
if not dbsOnly:
try:
blockInfo = self.phedex.getReplicaSEForBlocks(
phedexNodes=phedexNodes, block=blockNames, complete='y')
except Exception as ex:
msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockName
msg += "%s\n" % str(ex)
raise Exception(msg)
if not blockInfo or len(blockInfo) != len(
blockNames
): #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
dbsOnly = True
blockNames = set(blockNames) - set(
blockInfo
) #get the blocks we did not find information in phedex
if dbsOnly:
try:
for block in blockNames:
res = self.dbs.listBlockOrigin(block_name=block)
if res:
blockInfo[block] = [res[0]['origin_site_name']]
except dbsClientException as ex:
msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not any(blockInfo.values()): # no data location from dbs
return list()
#removing duplicates and 'UNKNOWN entries
locations = {}
node_filter_list = set(['UNKNOWN', None])
for name, nodes in blockInfo.iteritems():
final_nodes = set()
for n in nodes:
if n in node_filter_list:
continue
try:
cmsname(n)
except AssertionError: ## is SE
n = self.phedex.getNodeNames(n) if phedexNodes else [n]
else: ## not SE i.e. phedexNode
n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
final_nodes = final_nodes.union(n)
locations[name] = list(final_nodes - node_filter_list)
#returning single list if a single block is passed
if isinstance(fileBlockName, basestring):
locations = locations[fileBlockName]
return locations
def do_options():
op = OptionParser()
op.add_option("-u", "--url",
type="string",
action="store",
dest="couch",
help="CouchDB url. Default address http://127.0.0.1:5984",
default="http://127.0.0.1:5984")
op.add_option("-i", "--data",
dest="data",
default = [],
action="append",
type="string",
help="The name of a dataset(s) or block(s) you wish to stage")
op.add_option("-s", "--site",
dest="site",
default = [],
action="append",
type="string",
help="The T1 site(s) you want to stage the data at")
op.add_option("-v", "--verbose",
dest="verbose",
action="store_true",
default=False,
help="Be more verbose")
op.add_option("-d", "--debug",
dest="debug",
action="store_true",
default=False,
help="Be extremely verbose - print debugging statements")
op.add_option("--due",
dest="due",
default=False,
help="Add a due date to the request. Date format is DD/MM/YYYY")
options, args = op.parse_args()
logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger('StageManager')
if options.verbose:
logger.setLevel(logging.INFO)
if options.debug:
logger.setLevel(logging.DEBUG)
logger.info('options: %s, args: %s' % (options, args))
if options.due:
options.due = time.mktime(time.strptime(options.due, "%d/%m/%Y"))
if len(options.site) > 0:
for site in options.site:
if not cmsname(site):
logger.critical('%s is not a valid CMS name!' % site)
sys.exit(101)
if not site.startswith('T1'):
logger.critical('%s is not a T1 site' % site)
sys.exit(102)
else:
logger.critical('you need to provide a T1 site to stage at (-s option)')
sys.exit(103)
if len(options.data) == 0:
logger.critical('you need to provide some data (dataset or block name) to stage')
sys.exit(201)
if options.verbose:
logger.debug('Looks like some good input ya got yaself thar...')
return options, args, logger
