Esempio n. 1
0
    def listFileBlockLocation(self, fileBlockName, dbsOnly = False, phedexNodes=False):
        """
        _listFileBlockLocation_

        Get origin_site_name of a block

        """
        blockNames = [fileBlockName] if isinstance(fileBlockName, basestring) else fileBlockName
        for block in blockNames:
            self.checkBlockName(block)

        blockInfo = {}
        if not dbsOnly:
            try:
                blockInfo = self.phedex.getReplicaSEForBlocks(phedexNodes=phedexNodes, block=blockNames, complete='y')
            except Exception as ex:
                msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockName
                msg += "%s\n" % str(ex)
                raise Exception(msg)

            if not blockInfo or len(blockInfo) != len(blockNames): #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
                dbsOnly = True
                blockNames = set(blockNames) - set(blockInfo) #get the blocks we did not find information in phedex

        if dbsOnly:
            try:
                for block in blockNames:
                    res = self.dbs.listBlockOrigin(block_name = block)
                    if res:
                        blockInfo[block] = [res[0]['origin_site_name']]
            except dbsClientException as ex:
                msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockName
                msg += "%s\n" % formatEx3(ex)
                raise DBSReaderError(msg)

            if not any(blockInfo.values()): # no data location from dbs
                return list()

        #removing duplicates and 'UNKNOWN entries
        locations = {}
        node_filter_list = set(['UNKNOWN', None])
        for name, nodes in blockInfo.iteritems():
            final_nodes = set()
            for n in nodes:
                if n in node_filter_list:
                    continue
                try:
                    cmsname(n)
                except AssertionError: ## is SE
                    n = self.phedex.getNodeNames(n) if phedexNodes else [n]
                else:  ## not SE i.e. phedexNode
                    n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
                final_nodes = final_nodes.union(n)
            locations[name] = list(final_nodes - node_filter_list)

        #returning single list if a single block is passed
        if isinstance(fileBlockName, basestring):
            locations = locations[fileBlockName]

        return locations
Esempio n. 2
0
 def validate_name(self, input):
     if 'name' not in input.keys():
         if input['scheme'] == 'cms_name':
             input['name'] ='T%'
         else:
             input['name'] = 'CERN'  
     if input['scheme'] == 'cms_name' and input['name'] !='%':
         for name in self.makelist(input['name']):
             cmsname(name)
     elif input['scheme'] == 'resource':
         reg = "(\/|\/([\w#!:.?+=&%@!\-\/]))?"
         assert re.compile(reg).match(input['name']) != None , \
           "'%s' is not a valid FQDN (regexp: %s)" % (input['name'], reg)
     return input
Esempio n. 3
0
 def validate_name(self, input):
     if 'name' not in input.keys():
         if input['scheme'] == 'cms_name':
             input['name'] = 'T%'
         else:
             input['name'] = 'CERN'
     if input['scheme'] == 'cms_name' and input['name'] != '%':
         for name in self.makelist(input['name']):
             cmsname(name)
     elif input['scheme'] == 'resource':
         reg = "(\/|\/([\w#!:.?+=&%@!\-\/]))?"
         assert re.compile(reg).match(input['name']) != None , \
           "'%s' is not a valid FQDN (regexp: %s)" % (input['name'], reg)
     return input
Esempio n. 4
0
 def getWorkloadCreateArgs():
     baseArgs = StdBase.getWorkloadCreateArgs()
     specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
                 "InputDataset": {"optional": False, "validate": dataset, "null": False},
                 "ConfigCacheID": {"optional": True, "null": True},
                 "DataTier": {"default": "USER", "type": str,
                              "optional": True, "validate": None,
                              "attr": "dataTier", "null": False},
                 "PhysicsGroup": {"default": "", "optional": False,
                                  "null": False, "validate": physicsgroup},
                 "MergedLFNBase": {"default": "/store/results", "type": str,
                                   "optional": True, "validate": None,
                                   "attr": "mergedLFNBase", "null": False},
                 # site whitelist shouldn't be allowed, but let's make an exception for StoreResults
                 "SiteWhitelist": {"default": [], "type": makeNonEmptyList, "assign_optional": False,
                                   "validate": lambda x: all([cmsname(y) for y in x])},
                 "BlockBlacklist": {"default": [], "type": makeList,
                                    "optional": True, "validate": lambda x: all([block(y) for y in x]),
                                    "attr": "blockBlacklist", "null": False},
                 "BlockWhitelist": {"default": [], "type": makeList,
                                    "optional": True, "validate": lambda x: all([block(y) for y in x]),
                                    "attr": "blockWhitelist", "null": False},
                 "RunBlacklist": {"default": [], "type": makeList,
                                  "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
                                  "attr": "runBlacklist", "null": False},
                 "RunWhitelist": {"default": [], "type": makeList,
                                  "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
                                  "attr": "runWhitelist", "null": False}}
     baseArgs.update(specArgs)
     StdBase.setDefaultArgumentsProperty(baseArgs)
     return baseArgs
Esempio n. 5
0
def do_options(defaults):
    """
    Read the users arguments and set sensible defaults
    """
    # Configure the options from the passed in list
    op = OptionParser()
    for key in defaults:
        # Deep copy so we don't overwrite the default
        cur = copy.deepcopy(defaults[key])
        if cur['persist'] == True:
            cur['opts']['default'] = None
        op.add_option(key, cur['long'], **cur['opts'])

    # Run the option parser
    options, args = op.parse_args()

    # Configure the logger
    logging.basicConfig(level=logging.WARNING)
    logger = logging.getLogger('StageManager')
    if options.verbose:
        logger.setLevel(logging.INFO)
    if options.debug:
        logger.setLevel(logging.DEBUG)

    # Sanity checks
    logger.info('options: %s, args: %s' % (options, args))
    if not cmsname(options.site):
        logger.warning('%s is not a valid CMS name!' % options.site)
        sys.exit(101)
    elif not options.site.startswith('T1'):
        logger.warning('%s is not a T1 site' % options.site)
        sys.exit(102)

    return options, args, logger
Esempio n. 6
0
 def getWorkloadCreateArgs():
     baseArgs = StdBase.getWorkloadCreateArgs()
     specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
                 "InputDataset": {"optional": False, "validate": dataset, "null": False},
                 "ConfigCacheID": {"optional": True, "null": True},
                 "DataTier": {"default": "USER", "type": str,
                              "optional": True, "validate": None,
                              "attr": "dataTier", "null": False},
                 "PhysicsGroup": {"default": "", "optional": False,
                                  "null": False, "validate": physicsgroup},
                 "MergedLFNBase": {"default": "/store/results", "type": str,
                                   "optional": True, "validate": None,
                                   "attr": "mergedLFNBase", "null": False},
                 # site whitelist shouldn't be allowed, but let's make an exception for StoreResults
                 "SiteWhitelist": {"default": [], "type": makeNonEmptyList, "assign_optional": False,
                                   "validate": lambda x: all([cmsname(y) for y in x])},
                 "BlockBlacklist": {"default": [], "type": makeList,
                                    "optional": True, "validate": lambda x: all([block(y) for y in x]),
                                    "attr": "blockBlacklist", "null": False},
                 "BlockWhitelist": {"default": [], "type": makeList,
                                    "optional": True, "validate": lambda x: all([block(y) for y in x]),
                                    "attr": "blockWhitelist", "null": False},
                 "RunBlacklist": {"default": [], "type": makeList,
                                  "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
                                  "attr": "runBlacklist", "null": False},
                 "RunWhitelist": {"default": [], "type": makeList,
                                  "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
                                  "attr": "runWhitelist", "null": False}}
     baseArgs.update(specArgs)
     StdBase.setDefaultArgumentsProperty(baseArgs)
     return baseArgs
Esempio n. 7
0
    def getWorkloadCreateArgs():
        specArgs = {
            "RequestType": {
                "default": "Resubmission"
            },
            "ResubmissionCount": {
                "default": 1,
                "type": int
            },
            "OriginalRequestType": {
                "null": False
            },
            "OriginalRequestName": {
                "null": False
            },
            "InitialTaskPath": {
                "optional": False,
                "validate": lambda x: len(x.split('/')) > 2
            },
            "ACDCServer": {
                "default": "https://cmsweb.cern.ch/couchdb",
                "validate": couchurl,
                "attr": "acdcServer"
            },
            "ACDCDatabase": {
                "default": "acdcserver",
                "validate": identifier,
                "attr": "acdcDatabase"
            },
            "CollectionName": {
                "default": None,
                "null": True
            },
            "IgnoredOutputModules": {
                "default": [],
                "type": makeList
            },
            "SiteWhitelist": {
                "default": [],
                "type": makeList,
                "validate": lambda x: all([cmsname(y) for y in x])
            },
            # it can be Chained or MC requests, so lets make it optional
            "InputDataset": {
                "optional": True,
                "validate": dataset,
                "null": True
            }
        }

        StdBase.setDefaultArgumentsProperty(specArgs)
        return specArgs
Esempio n. 8
0
    def getWorkloadCreateArgs():
        specArgs = {"RequestType": {"default": "Resubmission"},
                    "OriginalRequestType": {"null": False},
                    "OriginalRequestName": {"null": False},
                    "InitialTaskPath": {"optional": False,
                                        "validate": lambda x: len(x.split('/')) > 2},
                    "ACDCServer": {"default": "https://cmsweb.cern.ch/couchdb", "validate": couchurl,
                                   "attr": "acdcServer"},
                    "ACDCDatabase": {"default": "acdcserver", "validate": identifier,
                                     "attr": "acdcDatabase"},
                    "CollectionName": {"default": None, "null": True},
                    "IgnoredOutputModules": {"default": [], "type": makeList},
                    "SiteWhitelist": {"default": [], "type": makeList,
                                      "validate": lambda x: all([cmsname(y) for y in x])},
                    # it can be Chained or MC requests, so lets make it optional
                    "InputDataset": {"optional": True, "validate": dataset, "null": True}}

        StdBase.setDefaultArgumentsProperty(specArgs)
        return specArgs
Esempio n. 9
0
    def getWorkloadArguments():
        """
        _getWorkloadArguments_

        This represents the authorative list of request arguments that are
        interpreted by the current spec class. 
        The list is formatted as a 2-level dictionary, the keys in the first level
        are the identifiers for the arguments processed by the current spec.
        The second level dictionary contains the information about that argument for
        validation:

        - default: Gives a default value if not provided,
                   this default value usually is good enough for a standard workflow. If the argument is not optional
                   and a default value is provided, this is only meant for test purposes.
        - type: A function that verifies the type of the argument, it may also cast it into the appropiate python type.    
                If the input is not compatible with the expected type, this method must throw an exception.
        - optional: This boolean value indicates if the value must be provided or not
        - validate: A function which validates the input after type casting,
                    it returns True if the input is valid, it can throw exceptions on invalid input.
        - attr: This represents the name of the attribute corresponding to the argument in the WMSpec object.
        - null: This indicates if the argument can have None as its value.
        Example:
        {
            RequestPriority : {'default' : 0,
                               'type' : int,
                               'optional' : False,
                               'validate' : lambda x : x > 0,
                               'attr' : 'priority',
                               'null' : False}
        }
        This replaces the old syntax in the __call__ of:

        self.priority = arguments.get("RequestPriority", 0)
        """
        arguments = {"RequestPriority": {"default" : 0, "type" : int,
                                         "optional" : False, "validate" : lambda x : (x >= 0 and x < 1e6),
                                         "attr" : "priority"},
                     "Requestor": {"default" : "unknown", "optional" : False,
                                   "attr" : "owner"},
                     "RequestorDN" : {"default" : "unknown", "optional" : False,
                                      "attr" : "owner_dn"},
                     "Group" : {"default" : "unknown", "optional" : False,
                                "attr" : "group"},
                     "VoGroup" : {"default" : "DEFAULT", "attr" : "owner_vogroup"},
                     "VoRole" : {"default" : "DEFAULT", "attr" : "owner_vorole"},
                     "AcquisitionEra" : {"default" : "None",  "attr" : "acquisitionEra", 
                                         "validate" : acqname},
                     "CMSSWVersion" : {"default" : "CMSSW_5_3_7", "validate" : cmsswversion,
                                       "optional" : False, "attr" : "frameworkVersion"},
                     "ScramArch" : {"default" : "slc5_amd64_gcc462", "optional" : False},
                     "ProcessingVersion" : {"default" : 0, "attr" : "processingVersion",
                                            "type" : int},
                     "ProcessingString" : {"default" : None, "attr" : "processingString",
                                           "null" : True},
                     "SiteBlacklist" : {"default" : [], "type" : makeList,
                                        "validate" : lambda x: all([cmsname(y) for y in x])},
                     "SiteWhitelist" : {"default" : [], "type" : makeList,
                                        "validate" : lambda x: all([cmsname(y) for y in x])},
                     "UnmergedLFNBase" : {"default" : "/store/unmerged"},
                     "MergedLFNBase" : {"default" : "/store/data"},
                     "MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
                                       "validate" : lambda x : x > 0},
                     "MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
                                       "validate" : lambda x : x > 0},
                     "MaxWaitTime" : {"default" : 24 * 3600, "type" : int,
                                      "validate" : lambda x : x > 0},
                     "MaxMergeEvents" : {"default" : 100000, "type" : int,
                                         "validate" : lambda x : x > 0},
                     "ValidStatus" : {"default" : "PRODUCTION"},
                     "DbsUrl" : {"default" : "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"},
                     "DashboardHost" : {"default" : "cms-wmagent-job.cern.ch"},
                     "DashboardPort" : {"default" : 8884, "type" : int,
                                        "validate" : lambda x : x > 0},
                     "OverrideCatalog" : {"default" : None, "null" : True},
                     "RunNumber" : {"default" : 0, "type" : int},
                     "TimePerEvent" : {"default" : 12.0, "type" : float,
                                       "optional" : False, "validate" : lambda x : x > 0},
                     "Memory" : {"default" : 2300.0, "type" : float,
                                 "optional" : False, "validate" : lambda x : x > 0},
                     "SizePerEvent" : {"default" : 512.0, "type" : float,
                                       "optional" : False, "validate" : lambda x : x > 0},
                     "PeriodicHarvestInterval" : {"default" : 0, "type" : int,
                                                  "validate" : lambda x : x >= 0},
                     "DQMUploadProxy" : {"default" : None, "null" : True,
                                         "attr" : "dqmUploadProxy"},
                     "DQMUploadUrl" : {"default" : "https://cmsweb.cern.ch/dqm/dev",
                                       "attr" : "dqmUploadUrl"},
                     "DQMSequences" : {"default" : [], "type" : makeList,
                                       "attr" : "dqmSequences"},
                     "DQMConfigCacheID" : {"default" : None, "null" : True,
                                           "attr" : "dqmConfigCacheID"},
                     "EnableHarvesting" : {"default" : False, "type" : strToBool},
                     "EnableNewStageout" : {"default" : False, "type" : strToBool},
                     "IncludeParents" : {"default" : False,  "type" : strToBool},
                     "Multicore" : {"default" : None, "null" : True,
                                    "validate" : lambda x : x == "auto" or (int(x) > 0)}}

        # Set defaults for the argument specification
        for arg in arguments:
            arguments[arg].setdefault("type", str)
            arguments[arg].setdefault("optional", True)
            arguments[arg].setdefault("null", False)
            arguments[arg].setdefault("validate", None)
            arguments[arg].setdefault("attr", arg[:1].lower() + arg[1:])

        return arguments
Esempio n. 10
0
                msg += "%s\n" % formatEx3(ex)
                raise DBSReaderError(msg)

            if not any(blockInfo.values()): # no data location from dbs
                return list()

        #removing duplicates and 'UNKNOWN entries
        locations = {}
        node_filter_list = set(['UNKNOWN', None])
        for name, nodes in blockInfo.iteritems():
            final_nodes = set()
            for n in nodes:
                if n in node_filter_list:
                    continue
                try:
                    cmsname(n)
                except AssertionError: ## is SE
                    n = self.phedex.getNodeNames(n) if phedexNodes else [n]
                else:  ## not SE i.e. phedexNode
                    n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
                final_nodes = final_nodes.union(n)
            locations[name] = list(final_nodes - node_filter_list)

        #returning single list if a single block is passed
        if isinstance(fileBlockName, basestring):
            locations = locations[fileBlockName]

        return locations

    def getFileBlock(self, fileBlockName):
        """
Esempio n. 11
0
    def getWorkloadArguments():
        """
        _getWorkloadArguments_

        This represents the authorative list of request arguments that are
        interpreted by the current spec class.
        The list is formatted as a 2-level dictionary, the keys in the first level
        are the identifiers for the arguments processed by the current spec.
        The second level dictionary contains the information about that argument for
        validation:

        - default: Gives a default value if not provided,
                   this default value usually is good enough for a standard workflow. If the argument is not optional
                   and a default value is provided, this is only meant for test purposes.
        - type: A function that verifies the type of the argument, it may also cast it into the appropiate python type.
                If the input is not compatible with the expected type, this method must throw an exception.
        - optional: This boolean value indicates if the value must be provided or not by user 
                    or inherited class can overwrite with default value.
        - assign_optional: This boolean value indicates if the value must be provided when workflow is assinged if False.
                    
        - validate: A function which validates the input after type casting,
                    it returns True if the input is valid, it can throw exceptions on invalid input.
        - attr: This represents the name of the attribute corresponding to the argument in the WMSpec object.
        - null: This indicates if the argument can have None as its value.
        
        If above is not specifyed, automatically set by following default value
        - default: None
        - type: str
        - optional: True
        - assign_optional: True
        - validate: None
        - attr: change first letter to lower case
        - null: False
        Example:
        {
            RequestPriority : {'default' : 0,
                               'type' : int,
                               'optional' : False,
                               'validate' : lambda x : x > 0,
                               'attr' : 'priority',
                               'null' : False}
        }
        This replaces the old syntax in the __call__ of:

        self.priority = arguments.get("RequestPriority", 0)
        """
        # if key is not specified it is set by default value
        
        arguments = {"RequestType" : {"optional" : False}, # this need to be overwritten by inherited class
                     "Requestor": {"default": "unknown", "attr" : "owner"},
                     "RequestorDN" : {"default": "unknown", "attr" : "owner_dn"},
                     "Group" : {"default": "unknown"},
                     "RequestPriority": {"default" : 8000, "type" : int, 
                                         "validate" : lambda x : (x >= 0 and x < 1e6),
                                         "attr" : "priority"},
                     "VoGroup" : {"default" : "unknown", "attr" : "owner_vogroup"},
                     "VoRole" : {"default" : "unknown", "attr" : "owner_vorole"},
                     "Campaign" : {"default" : ""},
                     "AcquisitionEra" : {"default" : "FAKE", "validate" : acqname, "assign_optional": False},
                     "CMSSWVersion" : {"validate" : cmsswversion,
                                       "optional" : False, "attr" : "frameworkVersion"},
                     "ScramArch" : {"default" : "slc5_amd64_gcc462", "optional" : False},
                     "GlobalTag" : {"null" : True},
                     "GlobalTagConnect" : {"null" : True},
                     "ProcessingVersion" : {"default" : 1, "type" : int},
                     "ProcessingString" : {"null" : True},
                     "LumiList" : {"default" : {}, "type" : makeLumiList},
                     "SiteBlacklist" : {"default" : [], "type" : makeList,
                                        "validate" : lambda x: all([cmsname(y) for y in x])},
                     "SiteWhitelist" : {"default" : [], "type" : makeList,
                                        "validate" : lambda x: all([cmsname(y) for y in x])},
                     "TrustSitelists" : {"default" : False, "type" : strToBool},
                     "UnmergedLFNBase" : {"default" : "/store/unmerged"},
                     "MergedLFNBase" : {"default" : "/store/data"},
                     "MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
                                       "validate" : lambda x : x > 0},
                     "MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
                                       "validate" : lambda x : x > 0},
                     "MaxWaitTime" : {"default" : 24 * 3600, "type" : int,
                                      "validate" : lambda x : x > 0},
                     "MaxMergeEvents" : {"default" : 100000, "type" : int,
                                         "validate" : lambda x : x > 0},
                     "ValidStatus" : {"default" : "PRODUCTION"},
                     "DbsUrl" : {"default" : "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
                                 "null" : True, "validate" : checkDBSURL},
                     "DashboardHost" : {"default" : "cms-wmagent-job.cern.ch"},
                     "DashboardPort" : {"default" : 8884, "type" : int,
                                        "validate" : lambda x : x > 0},
                     "OverrideCatalog" : {"null" : True},
                     "RunNumber" : {"default" : 0, "type" : int},
                     "TimePerEvent" : {"default" : 12.0, "type" : float,
                                       "validate" : lambda x : x > 0},
                     "Memory" : {"default" : 2300.0, "type" : float,
                                 "validate" : lambda x : x > 0},
                     "SizePerEvent" : {"default" : 512.0, "type" : float,
                                        "validate" : lambda x : x > 0},
                     "PeriodicHarvestInterval" : {"default" : 0, "type" : int,
                                                  "validate" : lambda x : x >= 0},
                     "DQMHarvestUnit" : {"default" : "byRun", "type" : str, "attr" : "dqmHarvestUnit"},
                     "DQMUploadProxy" : {"null" : True, "attr" : "dqmUploadProxy"},
                     "DQMUploadUrl" : {"default" : "https://cmsweb.cern.ch/dqm/dev", "attr" : "dqmUploadUrl"},
                     "DQMSequences" : {"default" : [], "type" : makeList, "attr" : "dqmSequences"},
                     "DQMConfigCacheID" : {"null" : True, "attr" : "dqmConfigCacheID"},
                     "EnableHarvesting" : {"default" : False, "type" : strToBool},
                     "EnableNewStageout" : {"default" : False, "type" : strToBool},
                     "IncludeParents" : {"default" : False,  "type" : strToBool},
                     "Multicore" : {"default" : 1, "null" : True,
                                    "validate" : lambda x : x == "auto" or (int(x) > 0)},
                     #from assignment: performance monitoring data
                     "MaxRSS" : {"default" : 2411724, "type" : int, "validate" : lambda x : x > 0},
                     "MaxVSize" : {"default" : 20411724, "type" : int, "validate" : lambda x : x > 0},
                     "SoftTimeout" : {"default" : 129600, "type" : int, "validate" : lambda x : x > 0},
                     "GracePeriod" : {"default" : 300, "type" : int, "validate" : lambda x : x > 0},
                     "UseSiteListAsLocation" : {"default" : False, "type" : bool},
                     
                     # Set phedex subscription information
                     "CustodialSites" : {"default" : [], "type" : makeList, "assign_optional": True,
                                         "validate" : lambda x: all([cmsname(y) for y in x])},
                     "NonCustodialSites" : {"default" : [], "type" : makeList, "assign_optional": True,
                                            "validate" : lambda x: all([cmsname(y) for y in x])},
                     "AutoApproveSubscriptionSites" : {"default" : [], "type" : makeList, "assign_optional": True, 
                                                       "validate" : lambda x: all([cmsname(y) for y in x])},
                     # should be Low, Normal, High
                     "SubscriptionPriority" : {"default" : "Low", "assign_optional": True,
                                               "validate" : lambda x: x in ["Low", "Normal", "High"]},
                     # should be Move Replica  
                     "CustodialSubType" : {"default" : "Replica", "type" : str, "assign_optional": True,
                                           "validate" : lambda x: x in ["Move", "Replica"]},
                     "NonCustodialSubType" : {"default" : "Replica", "type" : str, "assign_optional": True,
                                              "validate" : lambda x: x in ["Move", "Replica"]},
                     
                     # Block closing informaiont
                     "BlockCloseMaxWaitTime" : {"default" : 66400, "type" : int, "validate" : lambda x : x > 0},
                     "BlockCloseMaxFiles" : {"default" : 500, "type" : int, "validate" : lambda x : x > 0},
                     "BlockCloseMaxEvents" : {"default" : 25000000, "type" : int, "validate" : lambda x : x > 0},
                     "BlockCloseMaxSize" : {"default" : 5000000000000, "type" : int, "validate" : lambda x : x > 0},
                     
                     # dashboard activity
                     "Dashboard" : {"default": "", "type" : str},
                     # team name
                     "Team" : {"default": "", "type" : str},
                     "PrepID": {"default" : None, "null" : True}
                     }
       
        # Set defaults for the argument specification
        StdBase.setDefaultArgumentsProperty(arguments)

        return arguments
Esempio n. 12
0
                msg += "%s\n" % formatEx3(ex)
                raise DBSReaderError(msg)

            if not any(blockInfo.values()):  # no data location from dbs
                return list()

        #removing duplicates and 'UNKNOWN entries
        locations = {}
        node_filter_list = set(['UNKNOWN', None])
        for name, nodes in blockInfo.iteritems():
            final_nodes = set()
            for n in nodes:
                if n in node_filter_list:
                    continue
                try:
                    cmsname(n)
                except AssertionError:  ## is SE
                    n = self.phedex.getNodeNames(n) if phedexNodes else [n]
                else:  ## not SE i.e. phedexNode
                    n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
                final_nodes = final_nodes.union(n)
            locations[name] = list(final_nodes - node_filter_list)

        #returning single list if a single block is passed
        if isinstance(fileBlockName, basestring):
            locations = locations[fileBlockName]

        return locations

    def getFileBlock(self, fileBlockName):
        """
Esempio n. 13
0
def do_options():
    op = OptionParser()
    op.add_option("-u", "--url",
              type="string", 
              action="store", 
              dest="couch", 
              help="CouchDB url. Default address http://127.0.0.1:5985", 
              default="http://127.0.0.1:5985")
    
    op.add_option("-s", "--site",
              dest="site",
              default="", 
              help="The T1 site you want to query")

    op.add_option("-n", "--new",
              dest="new",
              action="store_true",
              default=False,
              help="Show status of new requests")
    
    op.add_option("-a", "--acquired",
              dest="acquired",
              action="store_true",
              default=False,
              help="Show status of acquired requests")
    
    op.add_option("-d", "--done",
              dest="done",
              action="store_true",
              default=False,
              help="Show status of done requests")
    
    op.add_option("-e", "--expired",
              dest="expired",
              action="store_true",
              default=False,
              help="Show status of expired requests")
    
    op.add_option("-q", "--no-detail",
              dest="nodetail", 
              action="store_true",
              default=False, 
              help="Hide the pre-requests information and show only totals")

    op.add_option("-v", "--data",
              dest="data",
              action="store_true",
              default=False,
              help="Show the data requested against each status")

    options, args = op.parse_args()
    
    logger = logging.getLogger('StageManager')
   
    if options.site != "":
        if not cmsname(options.site):
            logger.critical('%s is not a valid CMS name!' % options.site)
            sys.exit(101)
        if not options.site.startswith('T1'):
            logger.critical('%s is not a T1 site' % options.site)
            sys.exit(102)
    else:
        logger.critical('you need to provide a T1 site to stage at (-s option)')
        sys.exit(103)
        
    return options, args, logger
Esempio n. 14
0
    def getWorkloadArguments():
        """
        _getWorkloadArguments_

        This represents the authorative list of request arguments that are
        interpreted by the current spec class.
        The list is formatted as a 2-level dictionary, the keys in the first level
        are the identifiers for the arguments processed by the current spec.
        The second level dictionary contains the information about that argument for
        validation:

        - default: Gives a default value if not provided,
                   this default value usually is good enough for a standard workflow. If the argument is not optional
                   and a default value is provided, this is only meant for test purposes.
        - type: A function that verifies the type of the argument, it may also cast it into the appropiate python type.
                If the input is not compatible with the expected type, this method must throw an exception.
        - optional: This boolean value indicates if the value must be provided or not
        - validate: A function which validates the input after type casting,
                    it returns True if the input is valid, it can throw exceptions on invalid input.
        - attr: This represents the name of the attribute corresponding to the argument in the WMSpec object.
        - null: This indicates if the argument can have None as its value.
        Example:
        {
            RequestPriority : {'default' : 0,
                               'type' : int,
                               'optional' : False,
                               'validate' : lambda x : x > 0,
                               'attr' : 'priority',
                               'null' : False}
        }
        This replaces the old syntax in the __call__ of:

        self.priority = arguments.get("RequestPriority", 0)
        """
        arguments = {"RequestPriority": {"default" : 0, "type" : int,
                                         "optional" : False, "validate" : lambda x : (x >= 0 and x < 1e6),
                                         "attr" : "priority"},
                     "Requestor": {"default" : "unknown", "optional" : False,
                                   "attr" : "owner"},
                     "RequestorDN" : {"default" : "unknown", "optional" : False,
                                      "attr" : "owner_dn"},
                     "Group" : {"default" : "unknown", "optional" : False,
                                "attr" : "group"},
                     "VoGroup" : {"default" : "DEFAULT", "attr" : "owner_vogroup"},
                     "VoRole" : {"default" : "DEFAULT", "attr" : "owner_vorole"},
                     "AcquisitionEra" : {"default" : "None",  "attr" : "acquisitionEra",
                                         "validate" : acqname},
                     "CMSSWVersion" : {"default" : "CMSSW_5_3_7", "validate" : cmsswversion,
                                       "optional" : False, "attr" : "frameworkVersion"},
                     "ScramArch" : {"default" : "slc5_amd64_gcc462", "optional" : False},
                     "GlobalTag" : {"default" : None, "type" : str,
                                    "optional" : True, "validate" : None,
                                    "attr" : "globalTag", "null" : True},
                     "GlobalTagConnect" : {"default" : None, "type" : str,
                                           "optional" : True, "validate" : None,
                                           "attr" : "globalTagConnect", "null" : True},
                     "ProcessingVersion" : {"default" : 0, "attr" : "processingVersion",
                                            "type" : int},
                     "ProcessingString" : {"default" : None, "attr" : "processingString",
                                           "null" : True},
                     "LumiList" : {"default" : [], "type" : makeLumiList,
                                      "optional" : True, "validate" : None,
                                      "attr" : "lumiList", "null" : False},
                     "SiteBlacklist" : {"default" : [], "type" : makeList,
                                        "validate" : lambda x: all([cmsname(y) for y in x])},
                     "SiteWhitelist" : {"default" : [], "type" : makeList,
                                        "validate" : lambda x: all([cmsname(y) for y in x])},
                     "UnmergedLFNBase" : {"default" : "/store/unmerged"},
                     "MergedLFNBase" : {"default" : "/store/data"},
                     "MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
                                       "validate" : lambda x : x > 0},
                     "MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
                                       "validate" : lambda x : x > 0},
                     "MaxWaitTime" : {"default" : 24 * 3600, "type" : int,
                                      "validate" : lambda x : x > 0},
                     "MaxMergeEvents" : {"default" : 100000, "type" : int,
                                         "validate" : lambda x : x > 0},
                     "ValidStatus" : {"default" : "PRODUCTION"},
                     "DbsUrl" : {"default" : "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
                                 "null" : True, "validate" : checkDBSUrl},
                     "DashboardHost" : {"default" : "cms-wmagent-job.cern.ch"},
                     "DashboardPort" : {"default" : 8884, "type" : int,
                                        "validate" : lambda x : x > 0},
                     "OverrideCatalog" : {"default" : None, "null" : True},
                     "RunNumber" : {"default" : 0, "type" : int},
                     "TimePerEvent" : {"default" : 12.0, "type" : float,
                                       "optional" : False, "validate" : lambda x : x > 0},
                     "Memory" : {"default" : 2300.0, "type" : float,
                                 "optional" : False, "validate" : lambda x : x > 0},
                     "SizePerEvent" : {"default" : 512.0, "type" : float,
                                       "optional" : False, "validate" : lambda x : x > 0},
                     "PeriodicHarvestInterval" : {"default" : 0, "type" : int,
                                                  "validate" : lambda x : x >= 0},
                     "DQMUploadProxy" : {"default" : None, "null" : True,
                                         "attr" : "dqmUploadProxy"},
                     "DQMUploadUrl" : {"default" : "https://cmsweb.cern.ch/dqm/dev",
                                       "attr" : "dqmUploadUrl"},
                     "DQMSequences" : {"default" : [], "type" : makeList,
                                       "attr" : "dqmSequences"},
                     "DQMConfigCacheID" : {"default" : None, "null" : True,
                                           "attr" : "dqmConfigCacheID"},
                     "EnableHarvesting" : {"default" : False, "type" : strToBool},
                     "EnableNewStageout" : {"default" : False, "type" : strToBool},
                     "IncludeParents" : {"default" : False,  "type" : strToBool},
                     "Multicore" : {"default" : None, "null" : True,
                                    "validate" : lambda x : x == "auto" or (int(x) > 0)},
                     "PrepID": {"default" : None, "null" : True}}

        # Set defaults for the argument specification
        StdBase.setDefaultArgumentsProperty(arguments)

        return arguments
Esempio n. 15
0
    def listFileBlockLocation(self,
                              fileBlockName,
                              dbsOnly=False,
                              phedexNodes=False):
        """
        _listFileBlockLocation_

        Get origin_site_name of a block

        """
        blockNames = [fileBlockName] if isinstance(
            fileBlockName, basestring) else fileBlockName
        for block in blockNames:
            self.checkBlockName(block)

        blockInfo = {}
        if not dbsOnly:
            try:
                blockInfo = self.phedex.getReplicaSEForBlocks(
                    phedexNodes=phedexNodes, block=blockNames, complete='y')
            except Exception as ex:
                msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockName
                msg += "%s\n" % str(ex)
                raise Exception(msg)

            if not blockInfo or len(blockInfo) != len(
                    blockNames
            ):  #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
                dbsOnly = True
                blockNames = set(blockNames) - set(
                    blockInfo
                )  #get the blocks we did not find information in phedex

        if dbsOnly:
            try:
                for block in blockNames:
                    res = self.dbs.listBlockOrigin(block_name=block)
                    if res:
                        blockInfo[block] = [res[0]['origin_site_name']]
            except dbsClientException as ex:
                msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockName
                msg += "%s\n" % formatEx3(ex)
                raise DBSReaderError(msg)

            if not any(blockInfo.values()):  # no data location from dbs
                return list()

        #removing duplicates and 'UNKNOWN entries
        locations = {}
        node_filter_list = set(['UNKNOWN', None])
        for name, nodes in blockInfo.iteritems():
            final_nodes = set()
            for n in nodes:
                if n in node_filter_list:
                    continue
                try:
                    cmsname(n)
                except AssertionError:  ## is SE
                    n = self.phedex.getNodeNames(n) if phedexNodes else [n]
                else:  ## not SE i.e. phedexNode
                    n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
                final_nodes = final_nodes.union(n)
            locations[name] = list(final_nodes - node_filter_list)

        #returning single list if a single block is passed
        if isinstance(fileBlockName, basestring):
            locations = locations[fileBlockName]

        return locations
Esempio n. 16
0
def do_options():
    op = OptionParser()
    op.add_option("-u", "--url",
              type="string", 
              action="store", 
              dest="couch", 
              help="CouchDB url. Default address http://127.0.0.1:5984", 
              default="http://127.0.0.1:5984")
    
    op.add_option("-i", "--data",
              dest="data",
              default = [],
              action="append", 
              type="string", 
              help="The name of a dataset(s) or block(s) you wish to stage")
    
    op.add_option("-s", "--site",
              dest="site",
              default = [],
              action="append", 
              type="string", 
              help="The T1 site(s) you want to stage the data at")
    
    op.add_option("-v", "--verbose",
              dest="verbose", 
              action="store_true",
              default=False, 
              help="Be more verbose")
    op.add_option("-d", "--debug",
              dest="debug", 
              action="store_true",
              default=False, 
              help="Be extremely verbose - print debugging statements")
    op.add_option("--due", 
              dest="due",
              default=False,
              help="Add a due date to the request. Date format is DD/MM/YYYY")
    options, args = op.parse_args()
    
    logging.basicConfig(level=logging.WARNING)
    logger = logging.getLogger('StageManager')
    if options.verbose:
        logger.setLevel(logging.INFO)
    if options.debug:
        logger.setLevel(logging.DEBUG)

    logger.info('options: %s, args: %s' % (options, args))
    
    if options.due:
        options.due = time.mktime(time.strptime(options.due, "%d/%m/%Y"))
    
    if len(options.site) > 0:
        for site in options.site:
            if not cmsname(site):
                logger.critical('%s is not a valid CMS name!' % site)
                sys.exit(101)
            if not site.startswith('T1'):
                logger.critical('%s is not a T1 site' % site)
                sys.exit(102)
    else:
        logger.critical('you need to provide a T1 site to stage at (-s option)')
        sys.exit(103)
    if len(options.data) == 0:
        logger.critical('you need to provide some data (dataset or block name) to stage')
        sys.exit(201)
    if options.verbose:
        logger.debug('Looks like some good input ya got yaself thar...')
        
    return options, args, logger