Exemplo n.º 1
0
    def validateOptions(self):
        """
        Check if the sitelist parameter is a comma separater list of cms sitenames,
        and put the strings to be passed to the server to self
        """
        SubCommand.validateOptions(self)

        #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
        #server error handling we get:
        #    Server answered with: Invalid input parameter
        #    Reason is: Incorrect 'siteblacklist' parameter
        #which is not really user friendly.
        #Moreover, I prefer to be independent from Lexicon. I'll the regex here.
        sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$"  #sn_re => SiteName_RegularExpression
        sn_rec = re.compile(
            sn_re)  #sn_rec => SiteName_RegularExpressionCompiled

        self.sitewhitelist = ''
        self.siteblsklist = ''

        for siteList in ['sitewhitelist', 'siteblacklist']:
            result = ''
            paramVal = getattr(self.options, siteList, None)
            if paramVal:
                for site in paramVal.split(','):
                    if not sn_rec.match(site):
                        raise ConfigException(
                            "The sitename %s dows not look like a valid CMS name (not matching %s)"
                            % (site, sn_re))
                    result += "&%s=%s" % (siteList, site)
            setattr(self, siteList, result)
Exemplo n.º 2
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if hasattr(self.options, 'logpath') and self.options.logpath != None:
            if not path.isfile(self.options.logpath):
                msg = '%sError%s: Could not find the log file in the path: %s' % (
                    colors.RED, colors.NORMAL, self.options.logpath)
                raise ConfigurationException(msg)
        elif hasattr(self.options, 'task'):
            self.cmdconf['requiresTaskOption'] = True
            self.cmdconf['useCache'] = True
            if self.options.task == None:
                if len(self.args) == 1 and self.args[0]:
                    self.options.task = self.args[0]
                #for the case of 'crab uploadlog', user is using the .crab3 file
                if self.options.task == None and hasattr(self, 'crab3dic'):
                    if self.crab3dic["taskname"] != None:
                        self.options.task = self.crab3dic["taskname"]
                    else:
                        msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (
                            colors.RED, colors.NORMAL)
                        raise ConfigurationException(msg)
        else:
            msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (
                colors.RED, colors.NORMAL)
            raise ConfigurationException(msg)
Exemplo n.º 3
0
    def validateOptions(self):
        """
        Check if the sitelist parameter is a comma separater list of cms sitenames,
        and put the strings to be passed to the server to self
        """
        SubCommand.validateOptions(self)

        #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
        #server error handling we get:
        #    Server answered with: Invalid input parameter
        #    Reason is: Incorrect 'siteblacklist' parameter
        #which is not really user friendly.
        #Moreover, I prefer to be independent from Lexicon. I'll the regex here.
        sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
        sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled

        self.sitewhitelist = ''
        self.siteblsklist = ''

        for siteList in ['sitewhitelist', 'siteblacklist']:
            result = ''
            paramVal = getattr(self.options, siteList, None)
            if paramVal:
                for site in paramVal.split(','):
                    if not sn_rec.match(site):
                        raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) )
                    result += "&%s=%s" % (siteList, site)
            setattr(self, siteList, result)

        #check the format of jobids
        self.jobids = ''
        if getattr(self.options, 'jobids', None):
            self.jobids = validateJobids(self.options.jobids)
Exemplo n.º 4
0
    def validateOptions(self):
        #Figuring out the destination directory
        SubCommand.validateOptions(self)
        self.dest = None
        if self.options.outputpath is not None:
            if not os.path.isabs(self.options.outputpath):
                self.dest = os.path.abspath(self.options.outputpath)
            else:
                self.dest = self.options.outputpath

        #convert all to -1
        if getattr(self.options, 'quantity', None) == 'all':
            self.options.quantity = -1

        #check the format of jobids
        if getattr(self.options, 'jobids', None):
            if re.compile('^\d+(,\d+)*$').match(self.options.jobids):
                self.options.jobids = [
                    ('jobids', jobid)
                    for jobid in self.options.jobids.split(',')
                ]
            else:
                raise ConfigurationException(
                    "The command line option jobids should be a comma separated list of integers"
                )
Exemplo n.º 5
0
    def validateOptions(self):
        SubCommand.validateOptions(self)
        if self.options.idle and (self.options.long or self.options.summary):
            raise ConfigurationException(
                "Idle option (-i) conflicts with -u, and -l")
        self.json = self.options.json
        self.summary = self.options.summary
        self.idle = self.options.idle
        self.long = self.options.long

        acceptedsort = [
            "state", "site", "runtime", "memory", "cpu", "retries", "waste",
            "exitcode"
        ]
        if hasattr(self.options, 'sort') and self.options.sort != None:
            if not self.long:
                raise ConfigurationException(
                    '%sError%s: Please use option --long together with --sort'
                    % (colors.RED, colors.NORMAL))
            elif not self.options.sort in acceptedsort:
                raise ConfigurationException(
                    '%sError%s: Only these values are accepted for crab status --sort option: %s'
                    % (colors.RED, colors.NORMAL, acceptedsort))
            else:
                self.sort = self.options.sort
Exemplo n.º 6
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        #check the format of jobids
        self.jobids = ''
        if getattr(self.options, 'jobids', None):
            self.jobids = validateJobids(self.options.jobids)
Exemplo n.º 7
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.sitename is None:
            msg  = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL)
            msg += " Use the --site option."
            ex = MissingOptionException(msg)
            ex.missingOption = "sitename"
            raise ex
        if hasattr(self.options, 'command') and self.options.command != None:
            AvailableCommands = ['LCG', 'GFAL']
            self.command = self.options.command.upper()
            if self.command not in AvailableCommands:
                msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
                ex = ConfigurationException(msg)
                raise ex
        else:
            self.command = None
        if hasattr(self.options, 'checksum'):
            if re.match('^yes$|^no$', self.options.checksum):
                self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
            else:
                msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
                ex = ConfigurationException(msg)
                raise ex
Exemplo n.º 8
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if not hasattr(self.options,
                       'sitename') or self.options.sitename is None:
            self.logger.info("Missing site name, use '--site' option")
            raise MissingOptionException
Exemplo n.º 9
0
    def validateOptions(self):
        """
        After doing the general options validation from the parent SubCommand class,
        do the validation of options that are specific to the submit command.
        """

        ## First call validateOptions() from the SubCommand class.
        SubCommand.validateOptions(self)
        ## If no configuration file was passed as an option, try to extract it from the arguments.
        ## Assume that the arguments can only be:
        ##     1) the configuration file name, and
        ##     2) parameters to override in the configuration file.
        ## The last ones should all contain an '=' sign, so these are not candidates to be the
        ## configuration file argument. Also, the configuration file name should end with '.py'.
        ## If can not find a configuration file candidate, use the default 'crabConfig.py'.
        ## If find more than one candidate, raise ConfigurationException.

        if self.options.config is None:
            use_default = True
            if len(self.args):
                config_candidates = [(arg,i) for i,arg in enumerate(self.args) if '=' not in arg and arg[-3:] == '.py']
                config_candidate_names = set([config_candidate_name for (config_candidate_name,_) in config_candidates])
                if len(config_candidate_names) == 1:
                    self.options.config = config_candidates[0][0]
                    del self.args[config_candidates[0][1]]
                    use_default = False
                elif len(config_candidate_names) > 1:
                    self.logger.info('Unable to unambiguously extract the configuration file from the command-line arguments.')
                    self.logger.info('Possible candidates are: %s' % list(config_candidate_names))
                    raise ConfigurationException('ERROR: Unable to extract configuration file from command-line arguments.')
            if use_default:
                self.options.config = 'crabConfig.py'
Exemplo n.º 10
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        #check the format of jobids
        self.jobids = ''
        if getattr(self.options, 'jobids', None):
            self.jobids = validateJobids(self.options.jobids)
Exemplo n.º 11
0
    def validateOptions(self):
        #Figuring out the destination directory
        SubCommand.validateOptions(self)
        if self.options.outputpath is not None:
            if re.match("^[a-z]+://", self.options.outputpath):
                self.dest = self.options.outputpath
            elif not os.path.isabs( self.options.outputpath ):
                self.dest = os.path.abspath( self.options.outputpath )
            else:
                self.dest = self.options.outputpath

        #convert all to -1
        if getattr(self.options, 'quantity', None) == 'all':
            self.options.quantity = -1


        if hasattr(self.options, 'command') and self.options.command != None:
            AvailableCommands = ['LCG', 'GFAL']
            self.command = self.options.command.upper()
            if self.command not in AvailableCommands:
                msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
                ex = ConfigurationException(msg)
                raise ex
        else:
            self.command = None
        if hasattr(self.options, 'checksum'):
            if re.match('^yes$|^no$', self.options.checksum):
                self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
            else:
                msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
                ex = ConfigurationException(msg)
                raise ex
Exemplo n.º 12
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.scheddonly and self.options.cacheonly:
            self.logger.info(
                'Options --schedd and --cache can not be specified simultaneously. No purging will be done.'
            )
            raise ConfigException
Exemplo n.º 13
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.jobid is not None:
            try:
                int(self.options.jobid)
            except ValueError:
               raise ClientException("The --jobid option has to be an integer")
Exemplo n.º 14
0
    def validateOptions(self):
        """
        After doing the general options validation from the parent SubCommand class,
        do the validation of options that are specific to the submit command.
        """
        ## First do the basic validation in the SubCommand.
        SubCommand.validateOptions(self)

        validateSubmitOptions(self.options, self.args)
Exemplo n.º 15
0
    def validateOptions(self):
        """
        Check if the sitelist parameter is a comma separater list of cms sitenames,
        and put the strings to be passed to the server to self
        """
        SubCommand.validateOptions(self)

        #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
        #server error handling we get:
        #    Server answered with: Invalid input parameter
        #    Reason is: Incorrect 'siteblacklist' parameter
        #which is not really user friendly.
        #Moreover, I prefer to be independent from Lexicon. I'll the regex here.
        sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
        sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled

        self.sitewhitelist = ''
        self.siteblsklist = ''

        for siteList in ['sitewhitelist', 'siteblacklist']:
            result = ''
            paramVal = getattr(self.options, siteList, None)
            if paramVal:
                for site in paramVal.split(','):
                    if not sn_rec.match(site):
                        raise ConfigException("The sitename %s dows not look like a valid CMS name (not matching %s)" % (site, sn_re) )
                    result += "&%s=%s" % (siteList, site)
            setattr(self, siteList, result)

        #check the format of jobids
        self.jobids = ''
        if getattr(self.options, 'jobids', None):
            self.jobids = validateJobids(self.options.jobids)

        # Sanity checks for task sizes.  Limits are purposely fairly generous to provide some level of future-proofing.
        # The server may restrict further.
        self.numcores = None
        if self.options.numcores != None:
            if self.options.numcores < 1 or self.options.numcores > 128:
                raise ConfigException("The number of requested cores (%d) must be between 1 and 128." % (self.options.numcores))
            self.numcores = str(self.options.numcores)

        self.maxjobruntime = None
        if self.options.maxjobruntime != None:
            if self.options.maxjobruntime < 1 or self.options.maxjobruntime > 336:
                raise ConfigException("The requested max job runtime (%d hours) must be between 1 and 336 hours." % self.options.maxjobruntime)
            self.maxjobruntime = str(self.options.maxjobruntime)

        self.maxmemory = None
        if self.options.maxmemory != None:
            if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30:
                raise ConfigException("The requested per-job memory (%d MB) must be between 30 and 30720 MB." % self.options.maxmemory)
            self.maxmemory = str(self.options.maxmemory)

        self.priority = None
        if self.options.priority != None:
            self.priority = str(self.options.priority)
Exemplo n.º 16
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.sitename is None:
            msg  = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL)
            msg += " Use the --site option."
            ex = MissingOptionException(msg)
            ex.missingOption = "sitename"
            raise ex
Exemplo n.º 17
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.idle and (self.options.long or self.options.summary ):
            raise ConfigurationException("Idle option (-i) conflicts with -u, and -l")
        self.long = self.options.long
        self.json = self.options.json
        self.summary = self.options.summary
        self.idle = self.options.idle
Exemplo n.º 18
0
    def validateOptions(self):
        """
        After doing the general options validation from the parent SubCommand class,
        do the validation of options that are specific to the submit command.
        """
        ## First do the basic validation in the SubCommand.
        SubCommand.validateOptions(self)

        validateSubmitOptions(self.options, self.args)
Exemplo n.º 19
0
    def validateOptions(self):
        """
        Check if the output file is given and set as attribute
        """
        SubCommand.validateOptions(self)
        if not re.match('^yes$|^no$', self.options.usedbs):
            raise ConfigurationException("--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)")
        self.usedbs = 1 if self.options.usedbs=='yes' else 0

        setattr(self, 'outdir', getattr(self.options, 'outdir', None))
Exemplo n.º 20
0
 def validateOptions(self):
     SubCommand.validateOptions(self)
     if self.options.idle and (self.options.long or self.options.summary):
         raise ConfigurationException("Option --idle conflicts with --summary and --long")
     
     if self.options.sort is not None:
         sortOpts = ["state", "site", "runtime", "memory", "cpu", "retries", "waste", "exitcode"]
         if self.options.sort not in sortOpts:
             msg  = "%sError%s:" % (colors.RED, colors.NORMAL)
             msg += " Only the following values are accepted for --sort option: %s" % (sortOpts)
             raise ConfigurationException(msg)
Exemplo n.º 21
0
 def validateOptions(self):
     ## Do the options validation from SubCommand.
     try:
         SubCommand.validateOptions(self)
     except MissingOptionException as ex:
         if ex.missingOption == "task":
             msg  = "%sError%s:" % (colors.RED, colors.NORMAL)
             msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)"
             msg += " or to a log file (use the --logpath option)."
             ex = MissingOptionException(msg)
             ex.missingOption = "task"
         raise ex
Exemplo n.º 22
0
    def validateOptions(self):
        """
        Check if the output file is given and set as attribute
        """
        SubCommand.validateOptions(self)
        if not re.match('^yes$|^no$', self.options.usedbs):
            raise ConfigurationException(
                "--dbs option only accepts the yes and no values (--dbs=yes or --dbs=no)"
            )
        self.usedbs = 1 if self.options.usedbs == 'yes' else 0

        setattr(self, 'outdir', getattr(self.options, 'outdir', None))
Exemplo n.º 23
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.idle and (self.options.long or self.options.summary or self.options.publication):
            raise ConfigurationException("Idle option (-i) conflicts with -u, -l, and -p")
        if self.options.publication and (self.options.long or self.options.summary or self.options.idle):
            raise ConfigurationException("Publication option (-p) conflicts with -u, -l, and -i")
        self.long = self.options.long
        self.json = self.options.json
        self.summary = self.options.summary
        self.idle = self.options.idle
        self.publication = self.options.publication
Exemplo n.º 24
0
    def validateOptions(self):
        """
        Check if the sitelist parameter is a comma separater list of cms sitenames,
        and put the strings to be passed to the server to self
        """
        SubCommand.validateOptions(self)

        ## Check the format of the jobids option.
        if getattr(self.options, 'jobids'):
            jobidstuple = validateJobids(self.options.jobids)
            self.jobids = [str(jobid) for (_, jobid) in jobidstuple]

        #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
        #server error handling we get:
        #    Server answered with: Invalid input parameter
        #    Reason is: Incorrect 'siteblacklist' parameter
        #which is not really user friendly.
        #Moreover, I prefer to be independent from Lexicon. I'll the regex here.
        sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
        sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled
        for sitelist in ['sitewhitelist', 'siteblacklist']:
            if getattr(self.options, sitelist) is not None:
                for i, site_name in enumerate(getattr(self.options, sitelist).split(',')):
                    if not sn_rec.match(site_name):
                        msg  = "The site name %s does not look like a valid CMS site name" % (site_name)
                        msg += " (it is not matching the regular expression %s)." % (sn_re)
                        raise ConfigurationException(msg)
                setattr(self, sitelist, getattr(self.options, sitelist).split(','))

        ## Sanity checks for task sizes. Limits are purposely fairly generous to provide
        ## some level of future-proofing. The server may restrict further.
        if self.options.maxjobruntime is not None:
            if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60:
                msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime)
                raise ConfigurationException(msg)
            self.maxjobruntime = str(self.options.maxjobruntime)

        if self.options.maxmemory is not None:
            if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30:
                msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory)
                raise ConfigurationException(msg)
            self.maxmemory = str(self.options.maxmemory)

        if self.options.numcores is not None:
            if self.options.numcores < 1 or self.options.numcores > 128:
                msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores)
                raise ConfigurationException(msg)
            self.numcores = str(self.options.numcores)

        if self.options.priority is not None:
            self.priority = str(self.options.priority)
Exemplo n.º 25
0
    def validateOptions(self):
        """
        Check if the output file is given and set as attribute
        """
        SubCommand.validateOptions(self)

        if self.options.usedbs is not None:
            msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603."
            raise UnknownOptionException(msg)

        recoveryMethods = ['notFinished', 'notPublished', 'failed']
        if self.options.recovery not in recoveryMethods:
            msg  = "%sError%s:" % (colors.RED, colors.NORMAL)
            msg += " The --recovery option only accepts the following values: %s" % (recoveryMethods)
            raise ConfigurationException(msg)
Exemplo n.º 26
0
    def validateOptions(self):
        """
        Check if the output file is given and set as attribute
        """
        SubCommand.validateOptions(self)

        if self.options.usedbs is not None:
            msg = "CRAB command option error: the option --dbs has been deprecated since CRAB v3.3.1603."
            raise UnknownOptionException(msg)

        recoveryMethods = ['notFinished', 'notPublished', 'failed']
        if self.options.recovery not in recoveryMethods:
            msg = "%sError%s:" % (colors.RED, colors.NORMAL)
            msg += " The --recovery option only accepts the following values: %s" % (
                recoveryMethods)
            raise ConfigurationException(msg)
Exemplo n.º 27
0
    def validateOptions(self):
        SubCommand.validateOptions(self)
        if self.options.idle and (self.options.long or self.options.summary):
            raise ConfigurationException(
                "Option --idle conflicts with --summary and --long")

        if self.options.sort is not None:
            sortOpts = [
                "state", "site", "runtime", "memory", "cpu", "retries",
                "waste", "exitcode"
            ]
            if self.options.sort not in sortOpts:
                msg = "%sError%s:" % (colors.RED, colors.NORMAL)
                msg += " Only the following values are accepted for --sort option: %s" % (
                    sortOpts)
                raise ConfigurationException(msg)
Exemplo n.º 28
0
    def validateOptions(self):
        #Figuring out the destination directory
        SubCommand.validateOptions(self)
        self.dest = None
        if self.options.outputpath is not None:
            if not os.path.isabs( self.options.outputpath ):
                self.dest = os.path.abspath( self.options.outputpath )
            else:
                self.dest = self.options.outputpath

        #convert all to -1
        if getattr(self.options, 'quantity', None) == 'all':
            self.options.quantity = -1

        #check the format of jobids
        if getattr(self.options, 'jobids', None):
            self.options.jobids = validateJobids(self.options.jobids)
Exemplo n.º 29
0
    def validateOptions(self):
        SubCommand.validateOptions(self)
        if self.options.idle and (self.options.long or self.options.summary ):
            raise ConfigurationException("Idle option (-i) conflicts with -u, and -l")
        self.json = self.options.json
        self.summary = self.options.summary
        self.idle = self.options.idle
        self.long = self.options.long

        acceptedsort = ["state", "site", "runtime", "memory", "cpu", "retries", "waste"]
        if hasattr(self.options , 'sort') and self.options.sort != None:
            if not self.long:
                raise ConfigurationException('%sError%s: Please use option --long together with --sort' % (colors.RED, colors.NORMAL))
            elif not self.options.sort in acceptedsort:
                raise ConfigurationException('%sError%s: Only this value accepted for crab status sort: %s' % (colors.RED, colors.NORMAL, acceptedsort))
            else:
                self.sort = self.options.sort
Exemplo n.º 30
0
    def validateOptions(self):
        """
        After doing the general options validation from the parent SubCommand class,
        do the validation of options that are specific to the submit command.
        """

        ## First call validateOptions() from the SubCommand class.
        SubCommand.validateOptions(self)
        ## If no configuration file was passed as an option, try to extract it from the arguments.
        ## Assume that the arguments can only be:
        ##     1) the configuration file name, and
        ##     2) parameters to override in the configuration file.
        ## The last ones should all contain an '=' sign, so these are not candidates to be the
        ## configuration file argument. Also, the configuration file name should end with '.py'.
        ## If can not find a configuration file candidate, use the default 'crabConfig.py'.
        ## If find more than one candidate, raise ConfigurationException.

        if self.options.config is None:
            use_default = True
            if len(self.args):
                config_candidates = [(arg, i)
                                     for i, arg in enumerate(self.args)
                                     if '=' not in arg and arg[-3:] == '.py']
                config_candidate_names = set([
                    config_candidate_name
                    for (config_candidate_name, _) in config_candidates
                ])
                if len(config_candidate_names) == 1:
                    self.options.config = config_candidates[0][0]
                    del self.args[config_candidates[0][1]]
                    use_default = False
                elif len(config_candidate_names) > 1:
                    self.logger.info(
                        'Unable to unambiguously extract the configuration file from the command-line arguments.'
                    )
                    self.logger.info('Possible candidates are: %s' %
                                     list(config_candidate_names))
                    raise ConfigurationException(
                        'ERROR: Unable to extract configuration file from command-line arguments.'
                    )
            if use_default:
                self.options.config = 'crabConfig.py'
Exemplo n.º 31
0
    def validateOptions(self):
        #Figuring out the destination directory
        SubCommand.validateOptions(self)
        self.dest = None
        if self.options.outputpath is not None:
            if not os.path.isabs( self.options.outputpath ):
                self.dest = os.path.abspath( self.options.outputpath )
            else:
                self.dest = self.options.outputpath

        #convert all to -1
        if getattr(self.options, 'quantity', None) == 'all':
            self.options.quantity = -1

        #check the format of jobids
        if getattr(self.options, 'jobids', None):
            if re.compile('^\d+(,\d+)*$').match(self.options.jobids):
                self.options.jobids = [('jobids',jobid) for jobid in self.options.jobids.split(',')]
            else:
                raise ConfigurationException("The command line option jobids should be a comma separated list of integers")
Exemplo n.º 32
0
    def validateOptions(self):
        # Figuring out the destination directory
        SubCommand.validateOptions(self)
        self.dest = None
        if self.options.outputpath is not None:
            if re.match("^[a-z]+://", self.options.outputpath):
                self.dest = self.options.outputpath
            elif not os.path.isabs(self.options.outputpath):
                self.dest = os.path.abspath(self.options.outputpath)
            else:
                self.dest = self.options.outputpath

        # convert all to -1
        if getattr(self.options, "quantity", None) == "all":
            self.options.quantity = -1

        # check the format of jobids
        if getattr(self.options, "jobids", None):
            self.options.jobids = validateJobids(self.options.jobids)

        self.dump = self.options.dump
Exemplo n.º 33
0
    def validateOptions(self):
        #Figuring out the destination directory
        SubCommand.validateOptions(self)
        self.dest = None
        if self.options.outputpath is not None:
            if re.match("^[a-z]+://", self.options.outputpath):
                self.dest = self.options.outputpath
            elif not os.path.isabs(self.options.outputpath):
                self.dest = os.path.abspath(self.options.outputpath)
            else:
                self.dest = self.options.outputpath

        #convert all to -1
        if getattr(self.options, 'quantity', None) == 'all':
            self.options.quantity = -1

        #check the format of jobids
        if getattr(self.options, 'jobids', None):
            self.options.jobids = validateJobids(self.options.jobids)

        self.dump = self.options.dump
Exemplo n.º 34
0
    def validateOptions(self):

        ## Check if the --logpath option was used. If it was, don't require the task
        ## option.
        if self.options.logpath is not None:
            if not os.path.isfile(self.options.logpath):
                msg = "%sError%s:" % (colors.RED, colors.NORMAL)
                msg += " Could not find the log file %s." % (self.options.logpath)
                raise ConfigurationException(msg)
            self.cmdconf["requiresDirOption"] = False

        ## Do the options validation from SubCommand.
        try:
            SubCommand.validateOptions(self)
        except MissingOptionException as ex:
            if ex.missingOption == "task":
                msg = "%sError%s:" % (colors.RED, colors.NORMAL)
                msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)"
                msg += " or to a log file (use the --logpath option)."
                ex = MissingOptionException(msg)
                ex.missingOption = "task"
            raise ex
Exemplo n.º 35
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.sort is not None:
            sortOpts = [
                "state", "site", "runtime", "memory", "cpu", "retries",
                "waste", "exitcode"
            ]
            if self.options.sort not in sortOpts:
                msg = "%sError%s:" % (colors.RED, colors.NORMAL)
                msg += " Only the following values are accepted for --sort option: %s" % (
                    sortOpts)
                raise ConfigurationException(msg)

        if self.options.jobids:
            jobidstuple = validateJobids(self.options.jobids)
            self.jobids = [str(jobid) for (_, jobid) in jobidstuple]

        if self.options.jobids and not (self.options.long
                                        or self.options.sort):
            raise ConfigurationException(
                "Parameter --jobids can only be used in combination "
                "with --long or --sort options.")
Exemplo n.º 36
0
    def validateOptions(self):

        ## Check if the --logpath option was used. If it was, don't require the task
        ## option.
        if self.options.logpath is not None:
            if not os.path.isfile(self.options.logpath):
                msg = "%sError%s:" % (colors.RED, colors.NORMAL)
                msg += " Could not find the log file %s." % (
                    self.options.logpath)
                raise ConfigurationException(msg)
            self.cmdconf['requiresDirOption'] = False

        ## Do the options validation from SubCommand.
        try:
            SubCommand.validateOptions(self)
        except MissingOptionException as ex:
            if ex.missingOption == "task":
                msg = "%sError%s:" % (colors.RED, colors.NORMAL)
                msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)"
                msg += " or to a log file (use the --logpath option)."
                ex = MissingOptionException(msg)
                ex.missingOption = "task"
            raise ex
Exemplo n.º 37
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if hasattr(self.options, 'logpath') and self.options.logpath != None:
            if not path.isfile(self.options.logpath):
                msg = '%sError%s: Could not find the log file in the path: %s' % (colors.RED,colors.NORMAL,self.options.logpath)
                raise ConfigurationException(msg)
        elif hasattr(self.options, 'task'):
            self.cmdconf['requiresTaskOption'] = True
            self.cmdconf['useCache'] = True
            if self.options.task == None:
                if len(self.args) == 1 and self.args[0]:
                    self.options.task = self.args[0]
                #for the case of 'crab uploadlog', user is using the .crab3 file
                if self.options.task == None and hasattr(self, 'crab3dic'):
                    if  self.crab3dic["taskname"] != None:
                        self.options.task = self.crab3dic["taskname"]
                    else:
                        msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL)
                        raise ConfigurationException(msg)
        else:
            msg = '%sError%s: Please use the directory option -d or --logpath to specify which log to upload' % (colors.RED, colors.NORMAL)
            raise ConfigurationException(msg)
Exemplo n.º 38
0
    def validateOptions(self):
        #Figuring out the destination directory
        SubCommand.validateOptions(self)
        self.dest = None
        if self.options.outputpath is not None:
            if re.match("^[a-z]+://", self.options.outputpath):
                self.dest = self.options.outputpath
            elif not os.path.isabs( self.options.outputpath ):
                self.dest = os.path.abspath( self.options.outputpath )
            else:
                self.dest = self.options.outputpath

        #convert all to -1
        if getattr(self.options, 'quantity', None) == 'all':
            self.options.quantity = -1

        #check the format of jobids
        if getattr(self.options, 'jobids', None):
            self.options.jobids = validateJobids(self.options.jobids)

        if hasattr(self.options, 'command') and self.options.command != None:
            AvailableCommands = ['LCG', 'GFAL']
            self.command = self.options.command.upper()
            if self.command not in AvailableCommands:
                msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
                ex = ConfigurationException(msg)
                raise ex
        else:
            self.command = None
        if hasattr(self.options, 'checksum'):
            if re.match('^yes$|^no$', self.options.checksum):
                self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
            else:
                msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
                ex = ConfigurationException(msg)
                raise ex
Exemplo n.º 39
0
 def validateOptions(self):
     SubCommand.validateOptions(self)
Exemplo n.º 40
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if not hasattr(self.options, 'sitename') or self.options.sitename is None:
            self.logger.info("Missing site name, use '--site' option")
            raise MissingOptionException
Exemplo n.º 41
0
 def validateOptions(self):
     SubCommand.validateOptions(self)
Exemplo n.º 42
0
    def validateOptions(self):
        """
        Check if the sitelist parameter is a comma separater list of cms sitenames,
        and put the strings to be passed to the server to self
        """
        SubCommand.validateOptions(self)

        if self.options.publication:
            if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \
               self.options.maxjobruntime is not None or self.options.maxmemory is not None or \
               self.options.numcores is not None or self.options.priority is not None:
                msg  = "The options --sitewhitelist, --siteblacklist,"
                msg += " --maxjobruntime, --maxmemory, --numcores and  --priority"
                msg += " can not be specified together with the option --publication."
                msg += " The last option is to only resubmit (failed) publications,"
                msg += " in which case all of the first options make no sense."
                raise ConfigurationException(msg)
            if self.options.jobids:
                msg  = "The option --jobids"
                msg += " can not be specified together with the option --publication."
                msg += " The last option is to only resubmit (failed) publications,"
                msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)."
                raise ConfigurationException(msg)
            if self.options.force:
                msg  = "The option --force"
                msg += " can not be specified together with the option --publication."
                msg += " The last option is to only resubmit failed publications."
                msg += " Publications in a status other than 'failed' can not be resubmitted."
                raise ConfigurationException(msg)

        ## The --jobids option indicates which jobs have to be resubmitted. If it is not
        ## given, then all jobs in the task that are not running or successfully
        ## completed are resubmitted. If the user provides a list of job ids, then also
        ## successfully completed jobs can be resubmitted.

        ## Check the format of the jobids option.
        if self.options.jobids:
            jobidstuple = validateJobids(self.options.jobids)
            self.jobids = [str(jobid) for (_, jobid) in jobidstuple]

        ## The --force option should not be accepted unless combined with a user-given
        ## list of job ids via --jobids.
        if self.options.force and not self.jobids:
            msg = "Option --force can only be used in combination with option --jobids."
            raise ConfigurationException(msg)

        ## Covention used for the job parameters that the user can set when doing job
        ## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory,
        ## numcores and priority):
        ## - If the user doesn't set a parameter we don't pass it to the server and the
        ##   the server copies the original value the parameter had at task submission.
        ##   It copies it from the Task DB. Therefore we need to keep these parameters
        ##   in separate columns of the Task DB containing their original values.
        ## - For the site black- and whitelists, if the user passes an empty string,
        ##   e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the
        ##   server interprets this as and empty list ([]). If the user passes a given
        ##   list of sites, this new list overwrittes the original one.
        ## - The values of the parameters are used only for the resubmitted jobs (for
        ##   their first resubmission and all next automatic resubmissions).

        #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
        #server error handling we get:
        #    Server answered with: Invalid input parameter
        #    Reason is: Incorrect 'siteblacklist' parameter
        #which is not really user friendly.
        #Moreover, I prefer to be independent from Lexicon. I'll the regex here.
        sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$" #sn_re => SiteName_RegularExpression
        sn_rec = re.compile(sn_re) #sn_rec => SiteName_RegularExpressionCompiled
        for sitelist in ['sitewhitelist', 'siteblacklist']:
            if getattr(self.options, sitelist) is not None:
                if getattr(self.options, sitelist) != "":
                    for site_name in getattr(self.options, sitelist).split(','):
                        if '*' not in site_name and not sn_rec.match(site_name):
                            msg  = "The site name '%s' does not look like a valid CMS site name" % (site_name)
                            msg += " (it is not matching the regular expression '%s')." % (sn_re)
                            raise ConfigurationException(msg)
                    setattr(self, sitelist, getattr(self.options, sitelist).split(','))
                else:
                    setattr(self, sitelist, [])

        ## Sanity checks for task sizes. Limits are purposely fairly generous to provide
        ## some level of future-proofing. The server may restrict further.
        if self.options.maxjobruntime is not None:
            if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336*60:
                msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (self.options.maxjobruntime)
                raise ConfigurationException(msg)

        if self.options.maxmemory is not None:
            if self.options.maxmemory < 30 or self.options.maxmemory > 1024*30:
                msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (self.options.maxmemory)
                raise ConfigurationException(msg)

        if self.options.numcores is not None:
            if self.options.numcores < 1 or self.options.numcores > 128:
                msg = "The requested number of cores (%d) must be between 1 and 128." % (self.options.numcores)
                raise ConfigurationException(msg)

        if self.options.priority is not None:
            if self.options.priority < 1:
                msg = "The requested priority (%d) must be greater than 0." % (self.options.priority)
                raise ConfigurationException(msg)
Exemplo n.º 43
0
 def validateOptions(self):
     """
     Check if the output file is given and set as attribute
     """
     SubCommand.validateOptions(self)
     setattr(self, 'outdir', getattr(self.options, 'outdir', None))
Exemplo n.º 44
0
    def validateOptions(self):
        SubCommand.validateOptions(self)

        if self.options.scheddonly and self.options.cacheonly:
            self.logger.info('Options --schedd and --cache can not be specified simultaneously. No purging will be done.')
            raise ConfigException
Exemplo n.º 45
0
    def validateOptions(self):
        """
        Check if the sitelist parameter is a comma separater list of cms sitenames,
        and put the strings to be passed to the server to self
        """
        SubCommand.validateOptions(self)

        serverFactory = CRABClient.Emulator.getEmulator('rest')
        self.server = serverFactory(self.serverurl,
                                    self.proxyfilename,
                                    self.proxyfilename,
                                    version=__version__)
        uri = getUrl(self.instance, resource='task')
        crabDBInfo, _, _ = self.server.get(uri,
                                           data={
                                               'subresource':
                                               'search',
                                               'workflow':
                                               self.cachedinfo['RequestName']
                                           })
        self.splitting = getColumn(crabDBInfo, 'tm_split_algo')

        if self.options.publication:
            if self.options.sitewhitelist is not None or self.options.siteblacklist is not None or \
               self.options.maxjobruntime is not None or self.options.maxmemory is not None or \
               self.options.numcores is not None or self.options.priority is not None:
                msg = "The options --sitewhitelist, --siteblacklist,"
                msg += " --maxjobruntime, --maxmemory, --numcores and  --priority"
                msg += " can not be specified together with the option --publication."
                msg += " The last option is to only resubmit (failed) publications,"
                msg += " in which case all of the first options make no sense."
                raise ConfigurationException(msg)
            if self.options.jobids:
                msg = "The option --jobids"
                msg += " can not be specified together with the option --publication."
                msg += " The last option is to only resubmit (failed) publications,"
                msg += " which does not allow yet filtering on job ids (ALL failed publications will be resubmitted)."
                raise ConfigurationException(msg)
            if self.options.force:
                msg = "The option --force"
                msg += " can not be specified together with the option --publication."
                msg += " The last option is to only resubmit failed publications."
                msg += " Publications in a status other than 'failed' can not be resubmitted."
                raise ConfigurationException(msg)

        ## The --jobids option indicates which jobs have to be resubmitted. If it is not
        ## given, then all jobs in the task that are not running or successfully
        ## completed are resubmitted. If the user provides a list of job ids, then also
        ## successfully completed jobs can be resubmitted.

        ## Check the format of the jobids option.
        if self.options.jobids:
            jobidstuple = validateJobids(self.options.jobids,
                                         self.splitting != 'Automatic')
            self.jobids = [str(jobid) for (_, jobid) in jobidstuple]

        ## The --force option should not be accepted unless combined with a user-given
        ## list of job ids via --jobids.
        if self.options.force and not self.jobids:
            msg = "Option --force can only be used in combination with option --jobids."
            raise ConfigurationException(msg)

        ## Covention used for the job parameters that the user can set when doing job
        ## resubmission (i.e. siteblacklist, sitewhitelist, maxjobruntime, maxmemory,
        ## numcores and priority):
        ## - If the user doesn't set a parameter we don't pass it to the server and the
        ##   the server copies the original value the parameter had at task submission.
        ##   It copies it from the Task DB. Therefore we need to keep these parameters
        ##   in separate columns of the Task DB containing their original values.
        ## - For the site black- and whitelists, if the user passes an empty string,
        ##   e.g. --siteblacklist='', we pass to the server siteblacklist=empty and the
        ##   server interprets this as and empty list ([]). If the user passes a given
        ##   list of sites, this new list overwrittes the original one.
        ## - The values of the parameters are used only for the resubmitted jobs (for
        ##   their first resubmission and all next automatic resubmissions).

        #Checking if the sites provided by the user are valid cmsnames. Doing this because with only the
        #server error handling we get:
        #    Server answered with: Invalid input parameter
        #    Reason is: Incorrect 'siteblacklist' parameter
        #which is not really user friendly.
        #Moreover, I prefer to be independent from Lexicon. I'll the regex here.
        sn_re = "^T[1-3]_[A-Z]{2}(_[A-Za-z0-9]+)+$"  #sn_re => SiteName_RegularExpression
        sn_rec = re.compile(
            sn_re)  #sn_rec => SiteName_RegularExpressionCompiled
        for sitelist in ['sitewhitelist', 'siteblacklist']:
            if getattr(self.options, sitelist) is not None:
                if getattr(self.options, sitelist) != "":
                    for site_name in getattr(self.options,
                                             sitelist).split(','):
                        if '*' not in site_name and not sn_rec.match(
                                site_name):
                            msg = "The site name '%s' does not look like a valid CMS site name" % (
                                site_name)
                            msg += " (it is not matching the regular expression '%s')." % (
                                sn_re)
                            raise ConfigurationException(msg)
                    setattr(self, sitelist,
                            getattr(self.options, sitelist).split(','))
                else:
                    setattr(self, sitelist, [])

        ## Sanity checks for task sizes. Limits are purposely fairly generous to provide
        ## some level of future-proofing. The server may restrict further.
        if self.options.maxjobruntime is not None:
            if self.options.maxjobruntime < 60 or self.options.maxjobruntime > 336 * 60:
                msg = "The requested maximum job runtime (%d minutes) must be between 60 and 20160 minutes." % (
                    self.options.maxjobruntime)
                raise ConfigurationException(msg)

        if self.options.maxmemory is not None:
            if self.options.maxmemory < 30 or self.options.maxmemory > 1024 * 30:
                msg = "The requested per-job memory (%d MB) must be between 30 and 30720 MB." % (
                    self.options.maxmemory)
                raise ConfigurationException(msg)

        if self.options.numcores is not None:
            if self.options.numcores < 1 or self.options.numcores > 128:
                msg = "The requested number of cores (%d) must be between 1 and 128." % (
                    self.options.numcores)
                raise ConfigurationException(msg)

        if self.options.priority is not None:
            if self.options.priority < 1:
                msg = "The requested priority (%d) must be greater than 0." % (
                    self.options.priority)
                raise ConfigurationException(msg)
Exemplo n.º 46
0
 def validateOptions(self):
     """
     Check if the output file is given and set as attribute
     """
     SubCommand.validateOptions(self)
     setattr(self, 'outdir', getattr(self.options, 'outdir', None))