Beispiel #1
0
    def master_configure(self):

        exePath = fullpath(self.exe)

        if not os.path.isfile(exePath):
            raise ApplicationConfigurationError\
               ( None, "exe = '%s' not found" % self.exe )

        listFile = fullpath(self.imageList)
        if not os.path.isfile(listFile):
            raise ApplicationConfigurationError\
               ( None, "imageList file = '%s' not found" % self.imageList )

        imageFile = open(listFile)
        imageDir = fullpath(imageFile.readline().strip().rstrip(os.sep))
        inputList = imageFile.readlines()
        imageFile.close()

        if not imageDir:
            raise ApplicationConfigurationError\
               ( None, "Top-level image directory '%s' not found" % imageDir )

        outDir = fullpath(self.outDir)
        if not outDir:
            outDir = imageDir

        if not os.path.isdir(outDir):
            os.makedirs(outDir)

        imageList = []
        for filename in inputList:
            relativePath = filename.strip().strip(os.sep)
            imagePath = fullpath(os.path.join(imageDir, relativePath))
            if os.path.exists(imagePath):
                imageList.append(relativePath)
                resultDir = os.path.join(os.path.dirname(imagePath), "vseg")
                if os.path.isdir(resultDir):
                    shutil.rmtree(resultDir)
            else:
                logger.warning("Specified image file '%s' not found" %
                               imagePath)
                logger.warning("File will be skipped")

            if len(imageList) >= self.maxImage:
                if abs(self.maxImage) == self.maxImage:
                    break

        appDict = \
           {
           "exePath" : exePath,
           "imageDir" : imageDir,
           "outDir" : outDir,
           "imageList" : imageList
           }

        return (False, appDict)
Beispiel #2
0
   def master_configure( self ):

      listFile = fullpath( self.docList )
      if not os.path.isfile( listFile ):
         raise ApplicationConfigurationError\
            ( None, "docList file = '%s' not found" % self.docList )

      docFile = open( listFile )
      docDir = docFile.readline().strip().rstrip( os.sep )
      inputList = []

      if docDir: 
         docDir = fullpath( docDir )
         inputList = docFile.readlines()
         docFile.close()
         outDir = fullpath( self.outDir )
         if not outDir:
            outDir = docDir
         tagFile = ""
      else:
         outDir = fullpath( self.outDir )
         tagFile = listFile

      if not outDir:
         raise ApplicationConfigurationError\
            ( None, "Output directory not defined" )

      if not os.path.isdir( outDir ):
         os.makedirs( outDir )

      docList = []
      for input in inputList:
         url = input.split()[ 0 ]
         relativePath = input[ len( url ) : ].strip().strip( os.sep )
         docList.append( ( url, relativePath ) )

         if len( docList ) >= self.maxDoc:
            if abs( self.maxDoc ) == self.maxDoc:
               break

      if self.softwareDir:
        softwareDir = fullpath( self.softwareDir )
      else:
        softwareDir = ""

      appDict = \
         {
         "version" : self.version,
         "docDir" : docDir,
         "outDir" : outDir,
         "docList" : docList,
         "tagFile" : tagFile,
         "softwareDir" : softwareDir
         }

      return( False, appDict )
Beispiel #3
0
    def master_configure(self):

        listFile = fullpath(self.docList)
        if not os.path.isfile(listFile):
            raise ApplicationConfigurationError\
               ( None, "docList file = '%s' not found" % self.docList )

        docFile = open(listFile)
        docDir = docFile.readline().strip().rstrip(os.sep)
        inputList = []

        if docDir:
            docDir = fullpath(docDir)
            inputList = docFile.readlines()
            docFile.close()
            outDir = fullpath(self.outDir)
            if not outDir:
                outDir = docDir
            tagFile = ""
        else:
            outDir = fullpath(self.outDir)
            tagFile = listFile

        if not outDir:
            raise ApplicationConfigurationError\
               ( None, "Output directory not defined" )

        if not os.path.isdir(outDir):
            os.makedirs(outDir)

        docList = []
        for input in inputList:
            url = input.split()[0]
            relativePath = input[len(url):].strip().strip(os.sep)
            docList.append((url, relativePath))

            if len(docList) >= self.maxDoc:
                if abs(self.maxDoc) == self.maxDoc:
                    break

        if self.softwareDir:
            softwareDir = fullpath(self.softwareDir)
        else:
            softwareDir = ""

        appDict = \
           {
           "version" : self.version,
           "docDir" : docDir,
           "outDir" : outDir,
           "docList" : docList,
           "tagFile" : tagFile,
           "softwareDir" : softwareDir
           }

        return (False, appDict)
Beispiel #4
0
   def master_configure( self ):

      exePath = fullpath( self.exe )

      if not os.path.isfile( exePath ):
         raise ApplicationConfigurationError\
            ( None, "exe = '%s' not found" % self.exe )

      listFile = fullpath( self.imageList )
      if not os.path.isfile( listFile ):
         raise ApplicationConfigurationError\
            ( None, "imageList file = '%s' not found" % self.imageList )

      imageFile = open( listFile )
      imageDir = fullpath( imageFile.readline().strip().rstrip( os.sep ) )
      inputList = imageFile.readlines()
      imageFile.close()

      if not imageDir:
         raise ApplicationConfigurationError\
            ( None, "Top-level image directory '%s' not found" % imageDir )
      
      outDir = fullpath( self.outDir )
      if not outDir:
         outDir = imageDir

      if not os.path.isdir( outDir ):
         os.makedirs( outDir )

      imageList = []
      for filename in inputList:
         relativePath = filename.strip().strip( os.sep )
         imagePath = fullpath( os.path.join( imageDir, relativePath ) )
         if os.path.exists( imagePath ):
            imageList.append( relativePath )
            resultDir = os.path.join( os.path.dirname( imagePath ), "vseg" )
            if os.path.isdir( resultDir ):
               shutil.rmtree( resultDir )
         else:
            logger.warning( "Specified image file '%s' not found" % imagePath )
            logger.warning( "File will be skipped" )

         if len( imageList ) >= self.maxImage:
            if abs( self.maxImage ) == self.maxImage:
               break

      appDict = \
         {
         "exePath" : exePath,
         "imageDir" : imageDir,
         "outDir" : outDir,
         "imageList" : imageList
         }

      return( False, appDict )
Beispiel #5
0
 def _check_inputs(self):
     """Checks the validity of user's entries for BenderScript schema"""
     
     if not self.scripts and not self.imports : 
         raise ApplicationConfigurationError("Application scripts/imports are not defined")
     
     if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
     if isinstance ( self.imports , str ) : self.imports = [ File ( self.imports ) ]
     
     for f in self.scripts : f.name = fullpath ( f.name )
     for f in self.imports : f.name = fullpath ( f.name )
Beispiel #6
0
   def unpack( self, job = None, tarFile = None ):

      if not job:
         job = self._getParent()

      if not tarFile:
         tarFile = fullpath( os.path.join( job.outputdir, "%s.tar.gz" \
           % self.tarfile ) )

      if not os.path.exists( tarFile ):
         logger.warning( "Output not found for job %s" % job.id )
         logger.warning( "Setting job status to failed" )
         job.updateStatus( "failed" )
         return False

      outDir = job.application.outDir
      if not outDir:
         listFile = fullpath( job.application.imageList )
         if not os.path.isfile( listFile ):
            logger.warning( "imageList file = '%s' not found for job %s" % \
                  ( job.application.imageList, job.id ) )
            logger.warning( "Setting job status to failed" )
            job.updateStatus( "failed" )
            return False

         imageFile = open( listFile )
         job.application.outDir = imageFile.readline().strip().rstrip( os.sep )
         imageFile.close()

      outDir = fullpath( job.application.outDir )
      if not os.path.isdir( outDir ):
         os.makedirs( outDir )

      tmpStore = os.path.join( job.outputdir, "tmp" ) 
      if not os.path.isdir( tmpStore ):
         os.makedirs( tmpStore )

      untarCommand = "tar -C %s -zxf %s" % ( tmpStore, tarFile )
      status = os.system( untarCommand )
      if ( 0 != status ):
         logger.warning( "Problems untarring output for job %s" % job.id )
         logger.warning( "Setting job status to failed" )
         job.updateStatus( "failed" )
         return False

      copyCommand = "cp -r %s/* %s" % ( tmpStore, outDir )
      status = os.system( copyCommand )
      if ( 0 != status ):
         logger.warning( "Problems copying output for job %s" % job.id )
         logger.warning( "Setting job status to failed" )
         job.updateStatus( "failed" )
         return False

      return True
Beispiel #7
0
 def _check_inputs(self):
     """Checks the validity of user's entries for BenderScript schema"""
     
     if not self.scripts and not self.imports : 
         raise ApplicationConfigurationError("Application scripts/imports are not defined")
     
     if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
     if isinstance ( self.imports , str ) : self.imports = [ File ( self.imports ) ]
     
     for f in self.scripts : f.name = fullpath ( f.name )
     for f in self.imports : f.name = fullpath ( f.name )
Beispiel #8
0
    def _check_inputs(self):
        """Checks the validity of some of user's entries for Gaudi schema"""

        # Warn user that no optsfiles given
        if len(self.optsfile) == 0:
            logger.warning("The 'optsfile' is not set. I hope this is OK!")
        else:
            # Check for non-exting optsfiles defined
            nonexistentOptFiles = []
            for f in self.optsfile:
                from Ganga.GPIDev.Lib.File.File import File
                if type(f) is str:
                    myFile = File(f)
                else:
                    myFile = f
                myFile.name = fullpath(myFile.name)
                if not os.path.isfile(myFile.name):
                    nonexistentOptFiles.append(myFile)

            if len(nonexistentOptFiles):
                tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
                for _f in nonexistentOptFiles:
                    tmpmsg += "'%s', " % _f.name
                msg = tmpmsg[:-2] + ']'
                raise ApplicationConfigurationError(msg)
Beispiel #9
0
    def _check_inputs(self):
        """Checks the validity of some of user's entries for Gaudi schema"""

        # Warn user that no optsfiles given
        if len(self.optsfile) == 0:
            logger.warning("The 'optsfile' is not set. I hope this is OK!")
        else:
            # Check for non-exting optsfiles defined
            nonexistentOptFiles = []
            for f in self.optsfile:
                from Ganga.GPIDev.Lib.File.File import File
                if type(f) is str:
                    myFile = File(f)
                else:
                    myFile = f
                myFile.name = fullpath(myFile.name)
                if not os.path.isfile(myFile.name):
                    nonexistentOptFiles.append(myFile)

            if len(nonexistentOptFiles):
                tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
                for _f in nonexistentOptFiles:
                    tmpmsg += "'%s', " % _f.name
                msg = tmpmsg[:-2] + ']'
                raise ApplicationConfigurationError(msg)
Beispiel #10
0
   def download( self, job = None, tarFile = None ):

      if not job:
         job = self._getParent()

      if not tarFile:
         tarFile = fullpath( os.path.join( job.outputdir, "%s.tar.gz" \
           % self.tarfile ) )

      if ( "LCG" == job.backend._name ): 
         gridUrl = \
            os.path.join( job.outputdata.getGridStorage(), "%s.tar.gz" \
            % self.tarfile )
         cp = "globus-url-copy %s file:%s" % ( gridUrl, tarFile )
         status = shell.cmd1( cmd = cp, allowed_exit = range( 1000 ) )[ 0 ]
         if ( 0 != status ):
            logger.warning( "Problems retrieving output for job %s" % job.id )
            logger.warning( "Setting job status to failed" )
            job.updateStatus( "failed" )
            return False
         rm = "edg-gridftp-rm %s" % gridUrl
         shell.cmd1( cmd = rm, allowed_exit = range( 1000 ) )
         rmdir = "edg-gridftp-rmdir %s" % os.path.dirname( gridUrl )
         shell.cmd1( cmd = rmdir, allowed_exit = range( 1000 ) )

      return True
Beispiel #11
0
 def _check_inputs(self):
     """Checks the validity of user's entries for Ostap schema"""
     
     if not self.scripts and not self.commands and not self.arguments : 
         raise ApplicationConfigurationError(None, "Application scripts are not defined")
     
     if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]        
     for f in self.scripts : f.name = fullpath ( f.name )
Beispiel #12
0
    def _check_inputs(self):
        """Checks the validity of user's entries for Ostap schema"""

        if not self.scripts and not self.commands and not self.arguments:
            raise ApplicationConfigurationError(
                None, "Application scripts are not defined")

        if isinstance(self.scripts, str): self.scripts = [File(self.scripts)]
        for f in self.scripts:
            f.name = fullpath(f.name)
Beispiel #13
0
def checkDiskQuota():

    import subprocess
    from Ganga.Utility.files import fullpath

    repo_partition = getLocalRoot()
    repo_partition = os.path.realpath(repo_partition)

    work_partition = getLocalWorkspace()
    work_partition = os.path.realpath(work_partition)

    folders_to_check = [repo_partition, work_partition]

    home_dir = os.environ['HOME']
    to_remove = []
    for partition in folders_to_check:
        if not os.path.exists(partition):
            if home_dir not in folders_to_check:
                folders_to_check.append(home_dir)
            to_remove.append(partition)

    for folder in to_remove:
        folders_to_check.remove(folder)

    for data_partition in folders_to_check:

        if fullpath(data_partition).find('/afs') == 0:
            quota = subprocess.Popen(['fs', 'quota', '%s' % data_partition], stdout=subprocess.PIPE)
            output = quota.communicate()[0]
            logger.debug("fs quota %s:\t%s" % (str(data_partition), str(output)))
        else:
            df = subprocess.Popen(["df", '-Pk', data_partition], stdout=subprocess.PIPE)
            output = df.communicate()[0]

        try:
            global partition_warning
            global partition_critical
            quota_percent = output.split('%')[0]
            if int(quota_percent) >= partition_warning:
                logger.warning("WARNING: You're running low on disk space, Ganga may stall on launch or fail to download job output")
                logger.warning("WARNING: Please free some disk space on: %s" % str(data_partition))
            if int(quota_percent) >= partition_critical and config['force_start'] is False:
                logger.error("You are crtitically low on disk space!")
                logger.error("To prevent repository corruption and data loss we won't start Ganga.")
                logger.error("Either set your config variable 'force_start' in .gangarc to enable starting and ignore this check.")
                logger.error("Or, make sure you have more than %s percent free disk space on: %s" %(str(100-partition_critical), str(data_partition)))
                raise GangaException("Not Enough Disk Space!!!")
        except GangaException as err:
            raise err
        except Exception as err:
            logger.debug("Error checking disk partition: %s" % str(err))

    return
Beispiel #14
0
def checkDiskQuota():

    import subprocess

    repo_partition = getLocalRoot()
    repo_partition = os.path.realpath(repo_partition)

    work_partition = getLocalWorkspace()
    work_partition = os.path.realpath(work_partition)

    folders_to_check = [repo_partition, work_partition]

    home_dir = os.environ['HOME']
    to_remove = []
    for partition in folders_to_check:
        if not os.path.exists(partition):
            if home_dir not in folders_to_check:
                folders_to_check.append(home_dir)
            to_remove.append(partition)

    for folder in to_remove:
        folders_to_check.remove(folder)

    for data_partition in folders_to_check:

        if fullpath(data_partition, True).find('/afs') == 0:
            quota = subprocess.Popen(['fs', 'quota', '%s' % data_partition], stdout=subprocess.PIPE)
            output = quota.communicate()[0]
            logger.debug("fs quota %s:\t%s" % (str(data_partition), str(output)))
        else:
            df = subprocess.Popen(["df", '-Pk', data_partition], stdout=subprocess.PIPE)
            output = df.communicate()[0]

        try:
            global partition_warning
            global partition_critical
            quota_percent = output.split('%')[0]
            if int(quota_percent) >= partition_warning:
                logger.warning("WARNING: You're running low on disk space, Ganga may stall on launch or fail to download job output")
                logger.warning("WARNING: Please free some disk space on: %s" % str(data_partition))
            if int(quota_percent) >= partition_critical and config['force_start'] is False:
                logger.error("You are crtitically low on disk space!")
                logger.error("To prevent repository corruption and data loss we won't start Ganga.")
                logger.error("Either set your config variable 'force_start' in .gangarc to enable starting and ignore this check.")
                logger.error("Or, make sure you have more than %s percent free disk space on: %s" %(str(100-partition_critical), str(data_partition)))
                raise GangaException("Not Enough Disk Space!!!")
        except GangaException as err:
            raise err
        except Exception as err:
            logger.debug("Error checking disk partition: %s" % str(err))

    return
Beispiel #15
0
    def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):

        local = VansegLocal.prepare\
           ( self, app, appsubconfig, appmasterconfig, jobmasterconfig )
        inbox = []
        inbox.extend(local.inputbox)
        exeFile = File(fullpath(app.exe))
        if not exeFile in inbox:
            inbox.append(exeFile)
        requirements = LCGRequirements()
        lcg = LCGJobConfig( exe = local.exe, inputbox = inbox,\
           outputbox = local.outputbox, requirements = requirements )
        return lcg
Beispiel #16
0
    def _check_inputs(self):
        """Checks the validity of user's entries for GaudiPython schema"""
        if len(self.script) == 0:
            logger.warning("No script defined. Will use a default "
                           'script which is probably not what you want.')
            self.script = [File(os.path.join(
                os.path.dirname(inspect.getsourcefile(GaudiPython)),
                'options/GaudiPythonExample.py'))]
        else:
            for f in self.script:
                f.name = fullpath(f.name)

        return
Beispiel #17
0
   def prepare( self, app, appsubconfig, appmasterconfig, jobmasterconfig ):

      local = VansegLocal.prepare\
         ( self, app, appsubconfig, appmasterconfig, jobmasterconfig )
      inbox = []
      inbox.extend( local.inputbox )
      exeFile = File( fullpath( app.exe ) )
      if not exeFile in inbox:
         inbox.append( exeFile )
      requirements = LCGRequirements()
      lcg = LCGJobConfig( exe = local.exe, inputbox = inbox,\
         outputbox = local.outputbox, requirements = requirements )
      return lcg
Beispiel #18
0
 def _check_inputs(self):
     """Checks the validity of user's entries for GaudiPython schema"""
     # Always check for None OR empty
     if self.module.name == None:
         raise ApplicationConfigurationError(
             None, "Application Module not requested")
     elif self.module.name == "":
         raise ApplicationConfigurationError(
             None, "Application Module not requested")
     else:
         # Always check we've been given a FILE!
         self.module.name = fullpath(self.module.name)
         if not os.path.isfile(self.module.name):
             msg = 'Module file %s not found.' % self.module.name
             raise ApplicationConfigurationError(None, msg)
Beispiel #19
0
    def _check_inputs(self):
        """Checks the validity of user's entries for GaudiPython schema"""
        if len(self.script) == 0:
            logger.warning("No script defined. Will use a default "
                           'script which is probably not what you want.')
            self.script = [
                File(
                    os.path.join(
                        os.path.dirname(inspect.getsourcefile(GaudiPython)),
                        'options/GaudiPythonExample.py'))
            ]
        else:
            for f in self.script:
                f.name = fullpath(f.name)

        return
Beispiel #20
0
   def master_configure( self ):

      listFile = fullpath( self.imageList )
      if not os.path.isfile( listFile ):
         raise ApplicationConfigurationError\
            ( None, "imageList file = '%s' not found" % self.imageList )

      imageFile = open( listFile )
      imageDir = imageFile.readline().strip().rstrip( os.sep )
      inputList = []

      if imageDir: 
         imageDir = fullpath( imageDir )
         inputList = imageFile.readlines()
         imageFile.close()
         outDir = fullpath( self.outDir )
         if not outDir:
            outDir = imageDir
         tagFile = ""
      else:
         outDir = fullpath( self.outDir )
         tagFile = listFile

      if not outDir:
         raise ApplicationConfigurationError\
            ( None, "Output directory not defined" )

      if not os.path.isdir( outDir ):
         os.makedirs( outDir )

      imageList = []
      for input in inputList:
         url = input.split()[ 0 ]
         relativePath = input[ len( url ) : ].strip().strip( os.sep )
         imageList.append( ( url, relativePath ) )

         if len( imageList ) >= self.maxImage:
            if abs( self.maxImage ) == self.maxImage:
               break

      appDict = \
         {
         "version" : self.version,
         "classifierDir" : fullpath( self.classifierDir ),
         "libList" : [],
         "imageDir" : imageDir,
         "outDir" : outDir,
         "imageList" : imageList,
         "tagFile" : tagFile
         }
      for lib in self.libList:
         appDict[ "libList" ].append( fullpath( lib ) )

      return( False, appDict )
Beispiel #21
0
   def master_configure( self ):

      listFile = fullpath( self.imageList )
      if not os.path.isfile( listFile ):
         raise ApplicationConfigurationError\
            ( None, "imageList file = '%s' not found" % self.imageList )

      imageFile = open( listFile )
      imageDir = imageFile.readline().strip().rstrip( os.sep )
      inputList = []

      if imageDir: 
         imageDir = fullpath( imageDir )
         inputList = imageFile.readlines()
         imageFile.close()
         outDir = fullpath( self.outDir )
         if not outDir:
            outDir = imageDir
         tagFile = ""
      else:
         outDir = fullpath( self.outDir )
         tagFile = listFile

      if not outDir:
         raise ApplicationConfigurationError\
            ( None, "Output directory not defined" )

      if not os.path.isdir( outDir ):
         os.makedirs( outDir )

      imageList = []
      for input in inputList:
         url = input.split()[ 0 ]
         relativePath = input[ len( url ) : ].strip().strip( os.sep )
         imageList.append( ( url, relativePath ) )

         if len( imageList ) >= self.maxImage:
            if abs( self.maxImage ) == self.maxImage:
               break

      appDict = \
         {
         "version" : self.version,
         "classifierDir" : fullpath( self.classifierDir ),
         "libList" : [],
         "imageDir" : imageDir,
         "outDir" : outDir,
         "imageList" : imageList,
         "tagFile" : tagFile
         }
      for lib in self.libList:
         appDict[ "libList" ].append( fullpath( lib ) )

      return( False, appDict )
Beispiel #22
0
def checkDiskQuota():

    import subprocess
    from Ganga.Utility.files import fullpath

    repo_partition = getLocalRoot()
    repo_partition = os.path.realpath(repo_partition)

    work_partition = getLocalWorkspace()
    work_partition = os.path.realpath(work_partition)

    folders_to_check = [repo_partition, work_partition]

    home_dir = os.environ["HOME"]
    to_remove = []
    for partition in folders_to_check:
        if not os.path.exists(partition):
            if home_dir not in folders_to_check:
                folders_to_check.append(home_dir)
            to_remove.append(partition)

    for folder in to_remove:
        folders_to_check.remove(folder)

    for data_partition in folders_to_check:

        if fullpath(data_partition).find("/afs") == 0:
            quota = subprocess.Popen(["fs", "quota", "%s" % data_partition], stdout=subprocess.PIPE)
            output = quota.communicate()[0]
            logger.debug("fs quota %s:\t%s" % (str(data_partition), str(output)))

        else:
            df = subprocess.Popen(["df", "-Pk", data_partition], stdout=subprocess.PIPE)
            output = df.communicate()[0]

        try:
            quota_percent = output.split("%")[0]
            if int(quota_percent) >= 95:
                logger.warning(
                    "WARNING: You're running low on disk space, Ganga may stall on launch or fail to download job output"
                )
                logger.warning("WARNING: Please free some disk space on: %s" % str(data_partition))
        except Exception as err:
            logger.debug("Error checking disk partition: %s" % str(err))

    return
Beispiel #23
0
 def _check_inputs(self):
     """Checks the validity of user's entries for GaudiPython schema"""
     # Always check for None OR empty
     #logger.info("self.module: %s" % str(self.module))
     if isType(self.module, str):
         self.module = File(self.module)
     if self.module.name == None:
         raise ApplicationConfigurationError(
             "Application Module not requested")
     elif self.module.name == "":
         raise ApplicationConfigurationError(
             "Application Module not requested")
     else:
         # Always check we've been given a FILE!
         self.module.name = fullpath(self.module.name)
         if not os.path.isfile(self.module.name):
             msg = 'Module file %s not found.' % self.module.name
             raise ApplicationConfigurationError(msg)
Beispiel #24
0
    def __setattr__(self, attr, value):
        """
        This overloads the baseclass setter method and allows for dynamic evaluation of a parameter on assignment
        Args:
            attr (str): Name of the attribute which is being assigned for this class
            value (unknown): The raw value which is being passed to this class for assigning to the attribute
        """

        actual_value = value
        if attr == 'directory':
            if value:
                actual_value = path.abspath(fullpath(expandfilename(value)))
        elif attr == 'options':
            if isinstance(value, str):
                new_file = allComponentFilters['gangafiles'](value, None)
                actual_value = [ new_file ]
            elif isinstance(value, IGangaFile):
                actual_value = [ value ]
            elif not isinstance(value, (list, tuple, GangaList, type(None))):
                logger.warning("Possibly setting wrong type for options: '%s'" % type(value))

        super(GaudiExec, self).__setattr__(attr, actual_value)
Beispiel #25
0
   def fill( self ):

      hostname = commands.getoutput( "hostname -f" )
      job = self._getParent()

      tarFile = fullpath( os.path.join( job.outputdir, "%s.tar.gz" \
        % self.tarfile ) )

      downloadStartTime = "%.6f" % time.time()
      statusOK = self.download( job = job, tarFile = tarFile )
      downloadEndTime = "%.6f" % time.time()

      unpackStartTime = "%.6f" % time.time()
      if statusOK:
         statusOK = self.unpack( job = job, tarFile = tarFile )
      unpackEndTime = "%.6f" % time.time()

      cleanupStartTime = "%.6f" % time.time()
      if statusOK:
         statusOK = self.cleanup( job = job )
      cleanupEndTime = "%.6f" % time.time()

      lineList = \
         [
         "Hostname: %s" % hostname,
         "Download_start: %s" % downloadStartTime,
         "Download_end: %s" % downloadEndTime,
         "Unpack_start: %s" % unpackStartTime,
         "Unpack_end: %s" % unpackEndTime,
         "Cleanup_start: %s" % cleanupStartTime,
         "Cleanup_end: %s" % cleanupEndTime,
         ]

      outString = "\n".join( lineList )
      outfile = open( os.path.join( job.outputdir, "retrieve.dat" ), "w" )
      outfile.write( outString )
      outfile.close()

      return None
Beispiel #26
0
def requiresAfsToken():
    # Were we executed from within an AFS folder
    return fullpath(getLocalRoot(), True).find('/afs') == 0
Beispiel #27
0
    import platform
    import re
    c = re.compile('\S+-redhat-(?P<ver>\S+)-\S+')
    r = c.match(platform.platform())
    if r and r.group('ver').split('.')[0] == '5':
        platfstring = platf5

    return platfstring[arch]


# guess defaults if not defined
if not _defaultExternalHome:
    import os.path
    import Ganga
    from Ganga.Utility.files import fullpath
    p = fullpath(Ganga.__file__)
    for i in range(5):
        p = os.path.dirname(p)
    _defaultExternalHome = os.path.join(p, 'external')

if not _defaultPlatform:
    _defaultPlatform = detectPlatform()

from Ganga.Utility.Setup import PackageSetup
# The setup object
setup = PackageSetup(_externalPackages)


def standardSetup(setup=setup):
    """ Perform automatic initialization of the environment of the package.
    The gangaDir argument is only used by the core package, other packages should have no arguments.
Beispiel #28
0
    import platform
    import re
    c = re.compile('\S+-redhat-(?P<ver>\S+)-\S+')
    r = c.match(platform.platform())
    if r and r.group('ver').split('.')[0] == '5':
        platfstring = platf5

    return platfstring[arch]


# guess defaults if not defined
if not _defaultExternalHome:
    import os.path
    import Ganga
    from Ganga.Utility.files import fullpath
    p = fullpath(Ganga.__file__)
    for i in range(5):
        p = os.path.dirname(p)
    _defaultExternalHome = os.path.join(p, 'external')

if not _defaultPlatform:
    _defaultPlatform = detectPlatform()


from Ganga.Utility.Setup import PackageSetup
# The setup object
setup = PackageSetup(_externalPackages)


def standardSetup(setup=setup):
    """ Perform automatic initialization of the environment of the package.
Beispiel #29
0
def requiresAfsToken():
    return fullpath(getLocalRoot(), True).find('/afs') == 0
Beispiel #30
0
def get_user_dlls(appname,version,user_release_area,platform,shell):

  user_ra = user_release_area
  update_cmtproject_path(user_release_area)
  full_user_ra = fullpath(user_ra) # expand any symbolic links

  # Work our way through the CMTPATH until we find a cmt directory
  if not shell.env.has_key('CMTPATH'): return [], [], []
  projectdirs = shell.env['CMTPATH'].split(os.pathsep)
  #appveruser = os.path.join(appname + '_' + version,'cmt')
  appveruser = os.path.join(version,'cmt')
  #appverrelease = os.path.join(appname.upper(),appname.upper() + '_' + version,
  appverrelease = os.path.join('cmt')

  for projectdir in projectdirs:
    dir = fullpath(os.path.join(projectdir,appveruser))
    logger.error('Looking for projectdir %s' % dir)
    if os.path.exists(dir):
      break
    dir = fullpath(os.path.join(projectdir,appverrelease))
    logger.error('Looking for projectdir %s' % dir)
    if os.path.exists(dir):
      break

  logger.error('Using the CMT directory %s for identifying projects' % dir)
  rc, showProj, m = shell.cmd1('cd ' + dir +';cmt show projects', 
                               capture_stderr=True)

  logger.error(showProj)

  libs=[]
  merged_pys = []
  subdir_pys = {}
  project_areas = []
  py_project_areas = []

  for line in showProj.split('\n'):
    for entry in line.split():
      if entry.startswith(user_ra) or entry.startswith(full_user_ra):
        tmp = entry.rstrip('\)')
        libpath = fullpath(os.path.join(tmp,'InstallArea',platform,'lib'))
        logger.debug(libpath)
        project_areas.append(libpath)
        pypath = fullpath(os.path.join(tmp,'InstallArea','python'))
        logger.debug(pypath)
        py_project_areas.append(pypath)
        pypath = fullpath(os.path.join(tmp,'InstallArea',platform,'python'))
        logger.debug(pypath)
        py_project_areas.append(pypath)

  # savannah 47793 (remove multiple copies of the same areas)
  project_areas = unique(project_areas)
  py_project_areas = unique(py_project_areas)

  ld_lib_path = []
  if shell.env.has_key('LD_LIBRARY_PATH'):
    ld_lib_path = shell.env['LD_LIBRARY_PATH'].split(':')
  project_areas_dict = {}
  for area in project_areas:
    if area in ld_lib_path:
      project_areas_dict[area] = ld_lib_path.index(area)
    else:
      project_areas_dict[area] = 666
  from operator import itemgetter
  sorted_project_areas = []
  for item in sorted(project_areas_dict.items(),key=itemgetter(1)):
    sorted_project_areas.append(item[0])
  
  lib_names = []  
  for libpath in sorted_project_areas:
    if os.path.exists(libpath):
      for f in os.listdir(libpath):
        if lib_names.count(f) > 0: continue
        fpath = os.path.join(libpath,f)
        if os.path.exists(fpath):
          lib_names.append(f)
          libs.append(fpath)
        else:
          logger.warning("File %s in %s does not exist. Skipping...",
                         str(f),str(libpath))

  for pypath in py_project_areas:
    if os.path.exists( pypath):
      for f in os.listdir( pypath):
        confDB_path = os.path.join( pypath, f)
        if confDB_path.endswith( '.py'):
          if os.path.exists( confDB_path):
            merged_pys.append( confDB_path)
          else:
            logger.warning("File %s in %s does not exist. Skipping...",
                           str(f),str(confDB_path))
        elif os.path.isdir(confDB_path):
          pyfiles = []
          for g in os.listdir(confDB_path):
            file_path = os.path.join(confDB_path, g)
            if (file_path.endswith('.py')):
              if os.path.exists(file_path):
                pyfiles.append(file_path)
              else:
                logger.warning("File %s in %s does not exist. Skipping...",
                               str(g),str(f))
          subdir_pys[ f] = pyfiles
                    
  logger.debug("%s",pprint.pformat( libs))
  logger.debug("%s",pprint.pformat( merged_pys))
  logger.debug("%s",pprint.pformat( subdir_pys))

  return libs, merged_pys, subdir_pys
Beispiel #31
0
def getLocalRoot():
    from Ganga.Utility.files import fullpath
    import Ganga.Core.FileWorkspace
    return fullpath(Ganga.Core.FileWorkspace.gettop())
Beispiel #32
0
def requiresAfsToken():
    return fullpath(getLocalRoot(), True).find('/afs') == 0
Beispiel #33
0
def get_user_dlls(appname, version, user_release_area, platform, env):

    user_ra = user_release_area
    update_project_path(user_release_area)
    from Ganga.Utility.files import fullpath
    full_user_ra = fullpath(user_ra)  # expand any symbolic links

    # Work our way through the CMTPROJECTPATH until we find a cmt directory
    if 'CMTPROJECTPATH' not in env:
        return [], [], []
    projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
    appveruser = os.path.join(appname + '_' + version, 'cmt')
    appverrelease = os.path.join(appname.upper(),
                                 appname.upper() + '_' + version, 'cmt')

    for projectdir in projectdirs:
        projectDir = fullpath(os.path.join(projectdir, appveruser))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
        projectDir = fullpath(os.path.join(projectdir, appverrelease))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
    logger.debug('Using the CMT directory %s for identifying projects' %
                 projectDir)
    # rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
    # capture_stderr=True)
    from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
    rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)

    logger.debug(showProj)

    libs = []
    merged_pys = []
    subdir_pys = {}
    project_areas = []
    py_project_areas = []

    for line in showProj.split('\n'):
        for entry in line.split():
            if entry.startswith(user_ra) or entry.startswith(full_user_ra):
                tmp = entry.rstrip('\)')
                libpath = fullpath(
                    os.path.join(tmp, 'InstallArea', platform, 'lib'))
                logger.debug(libpath)
                project_areas.append(libpath)
                pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)
                pypath = fullpath(
                    os.path.join(tmp, 'InstallArea', platform, 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)

    # savannah 47793 (remove multiple copies of the same areas)
    from Ganga.Utility.util import unique
    project_areas = unique(project_areas)
    py_project_areas = unique(py_project_areas)

    ld_lib_path = []
    if 'LD_LIBRARY_PATH' in env:
        ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
        project_areas_dict = {}
    for area in project_areas:
        if area in ld_lib_path:
            project_areas_dict[area] = ld_lib_path.index(area)
        else:
            project_areas_dict[area] = 666
    from operator import itemgetter
    sorted_project_areas = []
    for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
        sorted_project_areas.append(item[0])

    lib_names = []
    for libpath in sorted_project_areas:
        if os.path.exists(libpath):
            for f in os.listdir(libpath):
                if lib_names.count(f) > 0:
                    continue
                fpath = os.path.join(libpath, f)
                if os.path.exists(fpath):
                    lib_names.append(f)
                    libs.append(fpath)
                else:
                    logger.warning("File %s in %s does not exist. Skipping...",
                                   str(f), str(libpath))

    for pypath in py_project_areas:
        if os.path.exists(pypath):
            from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
            from Ganga.Utility.Config import getConfig
            configGaudi = getConfig('GAUDI')
            pyFileCollector(pypath, merged_pys, subdir_pys,
                            configGaudi['pyFileCollectionDepth'])

    import pprint
    logger.debug("%s", pprint.pformat(libs))
    logger.debug("%s", pprint.pformat(merged_pys))
    logger.debug("%s", pprint.pformat(subdir_pys))

    return libs, merged_pys, subdir_pys
Beispiel #34
0
def requiresAfsToken():
    from Ganga.Utility.files import fullpath

    return fullpath(getLocalRoot()).find("/afs") == 0
Beispiel #35
0
def requiresAfsToken():
    # Were we executed from within an AFS folder
    return fullpath(getLocalRoot(), True).find('/afs') == 0
Beispiel #36
0
def get_user_dlls(appname, version, user_release_area, platform, env):

    user_ra = user_release_area
    update_project_path(user_release_area)
    from Ganga.Utility.files import fullpath
    full_user_ra = fullpath(user_ra)  # expand any symbolic links

    # Work our way through the CMTPROJECTPATH until we find a cmt directory
    if 'CMTPROJECTPATH' not in env:
        return [], [], []
    projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
    appveruser = os.path.join(appname + '_' + version, 'cmt')
    appverrelease = os.path.join(
        appname.upper(), appname.upper() + '_' + version, 'cmt')

    for projectdir in projectdirs:
        projectDir = fullpath(os.path.join(projectdir, appveruser))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
        projectDir = fullpath(os.path.join(projectdir, appverrelease))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
    logger.debug('Using the CMT directory %s for identifying projects' % projectDir)
    # rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
    # capture_stderr=True)
    from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
    rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)

    logger.debug(showProj)

    libs = []
    merged_pys = []
    subdir_pys = {}
    project_areas = []
    py_project_areas = []

    for line in showProj.split('\n'):
        for entry in line.split():
            if entry.startswith(user_ra) or entry.startswith(full_user_ra):
                tmp = entry.rstrip('\)')
                libpath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'lib'))
                logger.debug(libpath)
                project_areas.append(libpath)
                pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)
                pypath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)

    # savannah 47793 (remove multiple copies of the same areas)
    from Ganga.Utility.util import unique
    project_areas = unique(project_areas)
    py_project_areas = unique(py_project_areas)

    ld_lib_path = []
    if 'LD_LIBRARY_PATH' in env:
        ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
        project_areas_dict = {}
    for area in project_areas:
        if area in ld_lib_path:
            project_areas_dict[area] = ld_lib_path.index(area)
        else:
            project_areas_dict[area] = 666
    from operator import itemgetter
    sorted_project_areas = []
    for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
        sorted_project_areas.append(item[0])

    lib_names = []
    for libpath in sorted_project_areas:
        if os.path.exists(libpath):
            for f in os.listdir(libpath):
                if lib_names.count(f) > 0:
                    continue
                fpath = os.path.join(libpath, f)
                if os.path.exists(fpath):
                    lib_names.append(f)
                    libs.append(fpath)
                else:
                    logger.warning("File %s in %s does not exist. Skipping...", str(f), str(libpath))

    for pypath in py_project_areas:
        if os.path.exists(pypath):
            from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
            from Ganga.Utility.Config import getConfig
            configGaudi = getConfig('GAUDI')
            pyFileCollector(
                pypath, merged_pys, subdir_pys, configGaudi['pyFileCollectionDepth'])

    import pprint
    logger.debug("%s", pprint.pformat(libs))
    logger.debug("%s", pprint.pformat(merged_pys))
    logger.debug("%s", pprint.pformat(subdir_pys))

    return libs, merged_pys, subdir_pys