Ejemplo n.º 1
0
  def test_02_unparse( self ):
    """ pfnunparse and pfnunparse_old

    :param self: self reference
    """
    for pfn, result in self.pfns.items():
      if result["OK"]:
        self.assertEqual( pfnunparse( result["Value"] ), { "OK" : True, "Value" : pfn } )
    self.assertEqual( pfnunparse( None )['OK'], False )
    self.assertEqual( pfnunparse( "Path" )['OK'], False )
Ejemplo n.º 2
0
  def test_03_default_unparse( self ):
    """ pfnunparse and pfnunparse_old

    :param self: self reference
    """
    for pfn, result in self.default_pfns.items():
      if result["OK"]:
        unparseResult = pfnunparse( result["Value"], srmSpecific = False )
        self.assertEqual( unparseResult, { "OK" : True, "Value" : pfn } )
    self.assertEqual( pfnunparse( None )['OK'], False )
    self.assertEqual( pfnunparse( "Path" )['OK'], False )
Ejemplo n.º 3
0
  def test_03_srm_unparse( self ):
    """ pfnunparse and pfnunparse_old

    :param self: self reference
    """
    for pfn, result in self.srm_pfns.items():
      if result["OK"]:
        unparseResult = pfnunparse( result["Value"] )
        self.assertEqual( unparseResult, { "OK" : True, "Value" : pfn } )
    self.assertEqual( pfnunparse( None )['OK'], False )
    self.assertEqual( pfnunparse( "Path" )['OK'], False )
Ejemplo n.º 4
0
 def test_02_unparse( self ):
   """ pfnunparse and pfnunparse_old
    
   :param self: self reference
   """
   for pfn, result in self.pfns.items():
     if result["OK"]:
       self.assertEqual( pfnunparse( result["Value"] ), { "OK" : True, "Value" : pfn } )
   self.assertEqual( pfnunparse( None ), 
                     {'Message': "pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'NoneType'>", 'OK': False} )
   self.assertEqual( pfnunparse( "Path" ), 
                     {'Message': "pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'str'>", 'OK': False} )
Ejemplo n.º 5
0
    def constructURLFromLFN(self, lfn, withWSUrl=False):
        """Construct URL from the given LFN according to the VO convention for the
        primary protocol of the storage plagin

        :param str lfn: file LFN
        :param boolean withWSUrl: flag to include the web service part into the resulting URL
        :return result: result['Value'] - resulting URL
        """

        # Check the LFN convention:
        # 1. LFN must start with the VO name as the top level directory
        # 2. VO name must not appear as any subdirectory or file name
        lfnSplitList = lfn.split("/")
        voLFN = lfnSplitList[1]
        # TODO comparison to Sandbox below is for backward compatibility, should
        # be removed in the next release
        if voLFN != self.se.vo and voLFN != "SandBox" and voLFN != "Sandbox":
            return S_ERROR("LFN (%s) path must start with VO name (%s)" %
                           (lfn, self.se.vo))

        urlDict = dict(self.protocolParameters)
        urlDict["Options"] = "&".join(
            "%s=%s" % (optionName, urlDict[paramName])
            for paramName, optionName in self.DYNAMIC_OPTIONS.items()
            if urlDict.get(paramName))
        if not withWSUrl:
            urlDict["WSUrl"] = ""
        urlDict["FileName"] = lfn.lstrip("/")

        return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
Ejemplo n.º 6
0
  def constructURLFromLFN( self, lfn, withWSUrl = False ):
    """ Construct URL from the given LFN according to the VO convention for the
    primary protocol of the storage plagin

    :param str lfn: file LFN
    :param boolean withWSUrl: flag to include the web service part into the resulting URL
    :return result: result['Value'] - resulting URL
    """

    # Check the LFN convention:
    # 1. LFN must start with the VO name as the top level directory
    # 2. VO name must not appear as any subdirectory or file name
    lfnSplitList = lfn.split( '/' )
    voLFN = lfnSplitList[1]
    # TODO comparison to Sandbox below is for backward compatibility, should be removed in the next release
    if voLFN != self.se.vo and voLFN != "SandBox" and voLFN != "Sandbox":

      return S_ERROR( 'LFN does not follow the DIRAC naming convention %s' % lfn )

    urlDict = dict( self.protocolParameters )
    urlDict['Options'] = '&'.join( "%s=%s" % ( optionName, urlDict[paramName] )
                                   for paramName, optionName in self.DYNAMIC_OPTIONS.iteritems()
                                   if urlDict.get( paramName ) )
    if not withWSUrl:
      urlDict['WSUrl'] = ''
    urlDict['FileName'] = lfn.lstrip( '/' )

    return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
Ejemplo n.º 7
0
 def _resolvePFN(self, lfn, se):
   resSE = self.db.seManager.getSEDefinition(se)
   if not resSE['OK']:
     return resSE
   pfnDict = dict(resSE['Value']['SEDict'])
   if "PFNPrefix" in pfnDict:
     return S_OK(pfnDict['PFNPrefix'] + lfn)
   pfnDict['FileName'] = lfn
   return pfnunparse(pfnDict)
Ejemplo n.º 8
0
 def getProtocolPfn(self, pfnDict, withPort):
     """ From the pfn dict construct the pfn to be used
 """
     # pfnDict['Protocol'] = ''
     # pfnDict['Host'] = ''
     # pfnDict['Port'] = ''
     # pfnDict['WSUrl'] = ''
     res = pfnunparse(pfnDict)
     return res
Ejemplo n.º 9
0
 def getProtocolPfn( self, pfnDict, withPort ):
   """ From the pfn dict construct the pfn to be used
   """
   pfnDict['Protocol'] = ''
   pfnDict['Host'] = ''
   pfnDict['Port'] = ''
   pfnDict['WSUrl'] = ''
   res = pfnunparse( pfnDict )
   return res
Ejemplo n.º 10
0
 def getProtocolPfn(self, pfnDict, withPort):
     """ From the pfn dict construct the pfn to be used
 """
     pfnDict["Protocol"] = ""
     pfnDict["Host"] = ""
     pfnDict["Port"] = ""
     pfnDict["WSUrl"] = ""
     res = pfnunparse(pfnDict)
     return res
Ejemplo n.º 11
0
 def _resolvePFN(self, lfn, se):
     resSE = self.db.seManager.getSEDefinition(se)
     if not resSE["OK"]:
         return resSE
     pfnDict = dict(resSE["Value"]["SEDict"])
     if "PFNPrefix" in pfnDict:
         return S_OK(pfnDict["PFNPrefix"] + lfn)
     else:
         pfnDict["FileName"] = lfn
         return pfnunparse(pfnDict)
Ejemplo n.º 12
0
    def getURLBase(self, withWSUrl=False):
        """This will get the URL base. This is then appended with the LFN in DIRAC convention.

        :param self: self reference
        :param bool withWSUrl: flag to include Web Service part of the url
        :returns: URL
        """
        urlDict = dict(self.protocolParameters)
        if not withWSUrl:
            urlDict["WSUrl"] = ""
        return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
Ejemplo n.º 13
0
    def getEndpoint(self):
        """This will get endpoint of the storage. It basically is the same as :py:meth:`getURLBase`
            but without the basePath

        :returns: 'proto://hostname<:port>'

        """
        urlDict = dict(self.protocolParameters)
        # We remove the basePath
        urlDict["Path"] = ""
        return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
Ejemplo n.º 14
0
  def getURLBase( self, withWSUrl = False ):
    """ This will get the URL base. This is then appended with the LFN in DIRAC convention.

    :param self: self reference
    :param bool withWSUrl: flag to include Web Service part of the url
    :returns: URL
    """
    urlDict = dict( self.protocolParameters )
    if not withWSUrl:
      urlDict['WSUrl'] = ''
    return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
Ejemplo n.º 15
0
 def test_02_unparse(self):
     """ pfnunparse and pfnunparse_old
  
 :param self: self reference
 """
     for pfn, result in self.pfns.items():
         if result["OK"]:
             self.assertEqual(pfnunparse(result["Value"]), {
                 "OK": True,
                 "Value": pfn
             })
     self.assertEqual(
         pfnunparse(None), {
             'Message':
             "pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'NoneType'>",
             'OK': False
         })
     self.assertEqual(
         pfnunparse("Path"), {
             'Message':
             "pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'str'>",
             'OK': False
         })
Ejemplo n.º 16
0
    def getSEDefinition(self, seID):
        """ Get the Storage Element definition
    """
        if type(seID) in StringTypes:
            result = self.getSEID(seID)
            if not result['OK']:
                return result
            seID = result['Value']

        if seID in self.db.seDefinitions:
            if (time.time() - self.db.seDefinitions[seID]['LastUpdate']
                ) < self.seUpdatePeriod:
                if self.db.seDefinitions[seID]['SEDict']:
                    return S_OK(self.db.seDefinitions[seID])
            se = self.db.seDefinitions[seID]['SEName']
        else:
            result = self.getSEName(seID)
            if not result['OK']:
                return result
            se = result['Value']
            self.db.seDefinitions[seID] = {}
            self.db.seDefinitions[seID]['SEName'] = se
            self.db.seDefinitions[seID]['SEDict'] = {}
            self.db.seDefinitions[seID]['LastUpdate'] = 0.

        # We have to refresh the SE definition from the CS
        result = gConfig.getOptionsDict(
            '/Resources/StorageElements/%s/AccessProtocol.1' % se)
        if not result['OK']:
            return result
        seDict = result['Value']
        self.db.seDefinitions[seID]['SEDict'] = seDict
        if seDict:
            # A.T. Ports can be multiple, this can be better done using the Storage plugin
            # to provide the replica prefix to keep implementations in one place
            if 'Port' in seDict:
                ports = seDict['Port']
                if ',' in ports:
                    portList = [x.strip() for x in ports.split(',')]
                    random.shuffle(portList)
                    seDict['Port'] = portList[0]
            tmpDict = dict(seDict)
            tmpDict['FileName'] = ''
            result = pfnunparse(tmpDict)
            if result['OK']:
                self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result[
                    'Value']
        self.db.seDefinitions[seID]['LastUpdate'] = time.time()
        return S_OK(self.db.seDefinitions[seID])
Ejemplo n.º 17
0
    def updateURL(self, url, withWSUrl=False):
        """Update the URL according to the current SE parameters"""
        result = pfnparse(url, srmSpecific=self.srmSpecificParse)
        if not result["OK"]:
            return result
        urlDict = result["Value"]

        urlDict["Protocol"] = self.protocolParameters["Protocol"]
        urlDict["Host"] = self.protocolParameters["Host"]
        urlDict["Port"] = self.protocolParameters["Port"]
        urlDict["WSUrl"] = ""
        if withWSUrl:
            urlDict["WSUrl"] = self.protocolParameters["WSUrl"]

        return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
Ejemplo n.º 18
0
    def __addDoubleSlash(self, res):
        """ Utilities to add the double slash between the host(:port) and the path

        :param res: DIRAC return structure which contains an URL if S_OK
        :return: DIRAC structure with corrected URL
    """
        if not res['OK']:
            return res
        url = res['Value']
        res = pfnparse(url, srmSpecific=self.srmSpecificParse)
        if not res['OK']:
            return res
        urlDict = res['Value']
        urlDict['Path'] = '/' + urlDict['Path']
        return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
Ejemplo n.º 19
0
  def __addDoubleSlash( self, res ):
    """ Utilities to add the double slash between the host(:port) and the path

        :param res: DIRAC return structure which contains an URL if S_OK
        :return: DIRAC structure with corrected URL
    """
    if not res['OK']:
      return res
    url = res['Value']
    res = pfnparse( url, srmSpecific = self.srmSpecificParse )
    if not res['OK']:
      return res
    urlDict = res['Value']
    urlDict['Path'] = '/' + urlDict['Path']
    return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
Ejemplo n.º 20
0
  def getCurrentURL( self, fileName ):
    """ Obtain the current file URL from the current working directory and the filename

    :param self: self reference
    :param str fileName: path on storage
    """
    urlDict = dict( self.protocolParameters )
    if not fileName.startswith( '/' ):
      # Relative path is given
      urlDict['Path'] = self.cwd
    result = pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
    if not result['OK']:
      return result
    cwdUrl = result['Value']
    fullUrl = '%s%s' % ( cwdUrl, fileName )
    return S_OK( fullUrl )
Ejemplo n.º 21
0
    def getCurrentURL(self, fileName):
        """Obtain the current file URL from the current working directory and the filename

        :param self: self reference
        :param str fileName: path on storage
        """
        urlDict = dict(self.protocolParameters)
        if not fileName.startswith("/"):
            # Relative path is given
            urlDict["Path"] = self.cwd
        result = pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
        if not result["OK"]:
            return result
        cwdUrl = result["Value"]
        fullUrl = "%s%s" % (cwdUrl, fileName)
        return S_OK(fullUrl)
Ejemplo n.º 22
0
    def getCurrentURL(self, fileName):
        """ Obtain the current file URL from the current working directory and the filename

    :param self: self reference
    :param str fileName: path on storage
    """
        urlDict = dict(self.protocolParameters)
        if not fileName.startswith('/'):
            # Relative path is given
            urlDict['Path'] = self.cwd
        result = pfnunparse(urlDict)
        if not result['OK']:
            return result
        cwdUrl = result['Value']
        fullUrl = '%s%s' % (cwdUrl, fileName)
        return S_OK(fullUrl)
Ejemplo n.º 23
0
    def updateURL(self, url, withWSUrl=False):
        """ Update the URL according to the current SE parameters
    """
        result = pfnparse(url, srmSpecific=self.srmSpecificParse)
        if not result['OK']:
            return result
        urlDict = result['Value']

        urlDict['Protocol'] = self.protocolParameters['Protocol']
        urlDict['Host'] = self.protocolParameters['Host']
        urlDict['Port'] = self.protocolParameters['Port']
        urlDict['WSUrl'] = ''
        if withWSUrl:
            urlDict['WSUrl'] = self.protocolParameters['WSUrl']

        return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
Ejemplo n.º 24
0
  def updateURL( self, url, withWSUrl = False ):
    """ Update the URL according to the current SE parameters
    """
    result = pfnparse( url, srmSpecific = self.srmSpecificParse )
    if not result['OK']:
      return result
    urlDict = result['Value']

    urlDict['Protocol'] = self.protocolParameters['Protocol']
    urlDict['Host'] = self.protocolParameters['Host']
    urlDict['Port'] = self.protocolParameters['Port']
    urlDict['WSUrl'] = ''
    if withWSUrl:
      urlDict['WSUrl'] = self.protocolParameters['WSUrl']

    return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
Ejemplo n.º 25
0
  def getSEDefinition(self,seID):
    """ Get the Storage Element definition
    """
    if type(seID) in StringTypes:
      result = self.getSEID(seID)
      if not result['OK']:
        return result
      seID = result['Value']

    if seID in self.db.seDefinitions:
      if (time.time()-self.db.seDefinitions[seID]['LastUpdate']) < self.seUpdatePeriod:
        if self.db.seDefinitions[seID]['SEDict']:
          return S_OK(self.db.seDefinitions[seID])
      se = self.db.seDefinitions[seID]['SEName']  
    else:
      result = self.getSEName(seID)
      if not result['OK']:
        return result  
      se = result['Value']
      self.db.seDefinitions[seID] = {}
      self.db.seDefinitions[seID]['SEName'] = se
      self.db.seDefinitions[seID]['SEDict'] = {}
      self.db.seDefinitions[seID]['LastUpdate'] = 0.
      
    # We have to refresh the SE definition from the CS
    result = gConfig.getOptionsDict('/Resources/StorageElements/%s/AccessProtocol.1' % se)
    if not result['OK']:
      return result
    seDict = result['Value']
    self.db.seDefinitions[seID]['SEDict'] = seDict
    if seDict:
      # A.T. Ports can be multiple, this can be better done using the Storage plugin
      # to provide the replica prefix to keep implementations in one place
      if 'Port' in seDict:
        ports = seDict['Port']
        if ',' in ports:
          portList = [ x.strip() for x in ports.split(',') ]
          random.shuffle( portList )
          seDict['Port'] = portList[0]  
      tmpDict = dict(seDict)
      tmpDict['FileName'] = ''
      result = pfnunparse(tmpDict)
      if result['OK']:
        self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result['Value'] 
    self.db.seDefinitions[seID]['LastUpdate'] = time.time()
    return S_OK(self.db.seDefinitions[seID])
Ejemplo n.º 26
0
 def _checkLFNPFNConvention(self, lfn, pfn, se):
     """ Check that the PFN corresponds to the LFN-PFN convention
 """
     # Check if the PFN corresponds to the LFN convention
     if pfn == lfn:
         return S_OK()
     lfn_pfn = True  # flag that the lfn is contained in the pfn
     if (len(pfn) < len(lfn)) or (pfn[-len(lfn):] != lfn):
         return S_ERROR('PFN does not correspond to the LFN convention')
     if not pfn.endswith(lfn):
         return S_ERROR()
     # Check if the pfn corresponds to the SE definition
     result = self._getStorageElement(se)
     if not result['OK']:
         return result
     selement = result['Value']
     res = pfnparse(pfn)
     if not res['OK']:
         return res
     pfnDict = res['Value']
     protocol = pfnDict['Protocol']
     pfnpath = pfnDict['Path']
     result = selement.getStorageParameters(protocol)
     if not result['OK']:
         return result
     seDict = result['Value']
     sePath = seDict['Path']
     ind = pfnpath.find(sePath)
     if ind == -1:
         return S_ERROR(
             'The given PFN %s does not correspond to the %s SE definition'
             % (pfn, se))
     # Check the full LFN-PFN-SE convention
     lfn_pfn_se = True
     if lfn_pfn:
         seAccessDict = dict(seDict)
         seAccessDict['Path'] = sePath + '/' + lfn
         check_pfn = pfnunparse(seAccessDict)
         if check_pfn != pfn:
             return S_ERROR('PFN does not correspond to the LFN convention')
     return S_OK()
Ejemplo n.º 27
0
    def getSEDefinition(self, seID):
        """ Get the Storage Element definition
    """
        if type(seID) in StringTypes:
            result = self.getSEID(seID)
            if not result['OK']:
                return result
            seID = result['Value']

        if seID in self.db.seDefinitions:
            if (time.time() - self.db.seDefinitions[seID]['LastUpdate']
                ) < self.seUpdatePeriod:
                if self.db.seDefinitions[seID]['SEDict']:
                    return S_OK(self.db.seDefinitions[seID])
            se = self.db.seDefinitions[seID]['SEName']
        else:
            result = self.getSEName(seID)
            if not result['OK']:
                return result
            se = result['Value']
            self.db.seDefinitions[seID] = {}
            self.db.seDefinitions[seID]['SEName'] = se
            self.db.seDefinitions[seID]['SEDict'] = {}
            self.db.seDefinitions[seID]['LastUpdate'] = 0.

        # We have to refresh the SE definition from the CS
        result = gConfig.getOptionsDict(
            '/Resources/StorageElements/%s/AccessProtocol.1' % se)
        if not result['OK']:
            return result
        seDict = result['Value']
        self.db.seDefinitions[seID]['SEDict'] = seDict
        if seDict:
            tmpDict = dict(seDict)
            tmpDict['FileName'] = ''
            result = pfnunparse(tmpDict)
            if result['OK']:
                self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result[
                    'Value']
        self.db.seDefinitions[seID]['LastUpdate'] = time.time()
        return S_OK(self.db.seDefinitions[seID])
Ejemplo n.º 28
0
    def __putFile(self, src_file, dest_url):
        res = pfnparse(src_file)
        if not res['OK']:
            return res
        localCache = False
        srcDict = res['Value']
        if srcDict['Protocol'] in ['dips', 'dip']:
            # Make the service URL from the file URL by stripping off the file part
            serviceDict = dict(srcDict)
            serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
            serviceDict['FileName'] = ''
            res = pfnunparse(serviceDict)
            if not res['OK']:
                return res
            srcSEURL = res['Value']
            localCache = True
            transferClient = TransferClient(srcSEURL)
            res = transferClient.receiveFile(
                srcDict['FileName'],
                os.path.join(srcDict['Path'], srcDict['FileName']))
            if not res['OK']:
                return res
            src_file = srcDict['FileName']

        if not os.path.exists(src_file):
            errStr = "DIPStorage.__putFile: The source local file does not exist."
            gLogger.error(errStr, src_file)
            return S_ERROR(errStr)
        sourceSize = getSize(src_file)
        if sourceSize == -1:
            errStr = "DIPStorage.__putFile: Failed to get file size."
            gLogger.error(errStr, src_file)
            return S_ERROR(errStr)
        transferClient = TransferClient(self.url)
        res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
        if localCache:
            os.unlink(src_file)
        if res['OK']:
            return S_OK(sourceSize)
        else:
            return res
Ejemplo n.º 29
0
  def __putFile(self, src_file, dest_url):
    res = pfnparse(src_file)
    if not res['OK']:
      return res
    localCache = False
    srcDict = res['Value']
    if srcDict['Protocol'] in ['dips', 'dip']:
      # Make the service URL from the file URL by stripping off the file part
      serviceDict = dict(srcDict)
      serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
      serviceDict['FileName'] = ''
      res = pfnunparse(serviceDict)
      if not res['OK']:
        return res
      srcSEURL = res['Value']
      localCache = True
      transferClient = TransferClient(srcSEURL)
      res = transferClient.receiveFile(
          srcDict['FileName'], os.path.join(
              srcDict['Path'], srcDict['FileName']))
      if not res['OK']:
        return res
      src_file = srcDict['FileName']

    if not os.path.exists(src_file):
      errStr = "DIPStorage.__putFile: The source local file does not exist."
      gLogger.error(errStr, src_file)
      return S_ERROR(errStr)
    sourceSize = getSize(src_file)
    if sourceSize == -1:
      errStr = "DIPStorage.__putFile: Failed to get file size."
      gLogger.error(errStr, src_file)
      return S_ERROR(errStr)
    transferClient = TransferClient(self.url)
    res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
    if localCache:
      os.unlink(src_file)
    if res['OK']:
      return S_OK(sourceSize)
    else:
      return res
Ejemplo n.º 30
0
    def __init__(self, storageName, parameters):
        """ """
        StorageBase.__init__(self, storageName, parameters)
        self.pluginName = "DIP"

        self.log = gLogger.getSubLogger(self.__class__.__name__)

        # Several ports can be specified as comma separated list, choose
        # randomly one of those ports
        ports = self.protocolParameters["Port"].split(",")
        random.shuffle(ports)
        self.protocolParameters["Port"] = ports[0]

        pathDict = dict(self.protocolParameters)
        pathDict["Path"] = self.basePath
        result = pfnunparse(pathDict)
        if result["OK"]:
            self.url = result["Value"]

        self.checkSum = "CheckSum"
        self.isok = True
Ejemplo n.º 31
0
    def __init__(self, storageName, parameters):
        """
    """
        StorageBase.__init__(self, storageName, parameters)
        self.pluginName = 'DIP'

        self.log = gLogger.getSubLogger("DIPStorage", True)

        # Several ports can be specified as comma separated list, choose
        # randomly one of those ports
        ports = self.protocolParameters['Port'].split(',')
        random.shuffle(ports)
        self.protocolParameters['Port'] = ports[0]

        pathDict = dict(self.protocolParameters)
        pathDict['Path'] = self.basePath
        result = pfnunparse(pathDict)
        if result['OK']:
            self.url = result['Value']

        self.checkSum = "CheckSum"
        self.isok = True
Ejemplo n.º 32
0
 def getTransportURL( self, path, protocols = False ):
   """ Obtain the TURLs for the supplied path and protocols
   """
   res = self.exists( path )
   if res['OK']:
     for url in res['Value']['Successful']:
       if protocols and not self.protocol in protocols:
         res['Value']['Successful'].pop( url )
         res['Value']['Failed'][url] = 'Protocol not supported'
         continue
       if url[0] == '/':
         nameDict = self.getParameters()['Value']
         nameDict['FileName'] = url
         ret = pfnunparse( nameDict )
         if ret['OK']:
           res['Value']['Successful'][url] = ret['Value']
         else:
           res['Value']['Successful'].pop( url )
           res['Value']['Failed'][url] = ret['Message']
       else:
         res['Value']['Successful'][url] = url
   return res
Ejemplo n.º 33
0
 def getTransportURL(self, path, protocols=False):
     """ Obtain the TURLs for the supplied path and protocols
 """
     res = self.exists(path)
     if res['OK']:
         for url in res['Value']['Successful']:
             if protocols and not self.protocol in protocols:
                 res['Value']['Successful'].pop(url)
                 res['Value']['Failed'][url] = 'Protocol not supported'
                 continue
             if url[0] == '/':
                 nameDict = self.getParameters()['Value']
                 nameDict['FileName'] = url
                 ret = pfnunparse(nameDict)
                 if ret['OK']:
                     res['Value']['Successful'][url] = ret['Value']
                 else:
                     res['Value']['Successful'].pop(url)
                     res['Value']['Failed'][url] = ret['Message']
             else:
                 res['Value']['Successful'][url] = url
     return res
Ejemplo n.º 34
0
  def __init__( self, storageName, parameters ):
    """
    """
    StorageBase.__init__( self, storageName, parameters )
    self.pluginName = 'DIP'

    self.log = gLogger.getSubLogger( "DIPStorage", True )

    # Several ports can be specified as comma separated list, choose
    # randomly one of those ports
    ports = self.protocolParameters['Port'].split( ',' )
    random.shuffle( ports )
    self.protocolParameters['Port'] = ports[0]

    pathDict = dict( self.protocolParameters )
    pathDict['Path'] = self.basePath
    result = pfnunparse( pathDict )
    if result['OK']:
      self.url = result['Value']

    self.checkSum = "CheckSum"
    self.isok = True
Ejemplo n.º 35
0
 def _checkLFNPFNConvention(self, lfn, pfn, se):
     """ Check that the PFN corresponds to the LFN-PFN convention
 """
     # Check if the PFN corresponds to the LFN convention
     if pfn == lfn:
         return S_OK()
     lfn_pfn = True  # flag that the lfn is contained in the pfn
     if (len(pfn) < len(lfn)) or (pfn[-len(lfn) :] != lfn):
         return S_ERROR("PFN does not correspond to the LFN convention")
     if not pfn.endswith(lfn):
         return S_ERROR()
     # Check if the pfn corresponds to the SE definition
     result = self._getStorageElement(se)
     if not result["OK"]:
         return result
     selement = result["Value"]
     res = pfnparse(pfn)
     if not res["OK"]:
         return res
     pfnDict = res["Value"]
     protocol = pfnDict["Protocol"]
     pfnpath = pfnDict["Path"]
     result = selement.getStorageParameters(protocol)
     if not result["OK"]:
         return result
     seDict = result["Value"]
     sePath = seDict["Path"]
     ind = pfnpath.find(sePath)
     if ind == -1:
         return S_ERROR("The given PFN %s does not correspond to the %s SE definition" % (pfn, se))
     # Check the full LFN-PFN-SE convention
     lfn_pfn_se = True
     if lfn_pfn:
         seAccessDict = dict(seDict)
         seAccessDict["Path"] = sePath + "/" + lfn
         check_pfn = pfnunparse(seAccessDict)
         if check_pfn != pfn:
             return S_ERROR("PFN does not correspond to the LFN convention")
     return S_OK()
Ejemplo n.º 36
0
  def getSEDefinition(self,seID):
    """ Get the Storage Element definition
    """
    if type(seID) in StringTypes:
      result = self.getSEID(seID)
      if not result['OK']:
        return result
      seID = result['Value']

    if seID in self.db.seDefinitions:
      if (time.time()-self.db.seDefinitions[seID]['LastUpdate']) < self.seUpdatePeriod:
        if self.db.seDefinitions[seID]['SEDict']:
          return S_OK(self.db.seDefinitions[seID])
      se = self.db.seDefinitions[seID]['SEName']  
    else:
      result = self.getSEName(seID)
      if not result['OK']:
        return result  
      se = result['Value']
      self.db.seDefinitions[seID] = {}
      self.db.seDefinitions[seID]['SEName'] = se
      self.db.seDefinitions[seID]['SEDict'] = {}
      self.db.seDefinitions[seID]['LastUpdate'] = 0.
      
    # We have to refresh the SE definition from the CS
    result = gConfig.getOptionsDict('/Resources/StorageElements/%s/AccessProtocol.1' % se)
    if not result['OK']:
      return result
    seDict = result['Value']
    self.db.seDefinitions[seID]['SEDict'] = seDict
    if seDict:
      tmpDict = dict(seDict)
      tmpDict['FileName'] = ''
      result = pfnunparse(tmpDict)
      if result['OK']:
        self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result['Value'] 
    self.db.seDefinitions[seID]['LastUpdate'] = time.time()
    return S_OK(self.db.seDefinitions[seID])
Ejemplo n.º 37
0
    def getSEDefinition(self, seID):
        """ Get the Storage Element definition
    """
        if type(seID) in StringTypes:
            result = self.getSEID(seID)
            if not result["OK"]:
                return result
            seID = result["Value"]

        if seID in self.db.seDefinitions:
            if (time.time() - self.db.seDefinitions[seID]["LastUpdate"]) < self.seUpdatePeriod:
                if self.db.seDefinitions[seID]["SEDict"]:
                    return S_OK(self.db.seDefinitions[seID])
            se = self.db.seDefinitions[seID]["SEName"]
        else:
            result = self.getSEName(seID)
            if not result["OK"]:
                return result
            se = result["Value"]
            self.db.seDefinitions[seID] = {}
            self.db.seDefinitions[seID]["SEName"] = se
            self.db.seDefinitions[seID]["SEDict"] = {}
            self.db.seDefinitions[seID]["LastUpdate"] = 0.0

        # We have to refresh the SE definition from the CS
        result = gConfig.getOptionsDict("/Resources/StorageElements/%s/AccessProtocol.1" % se)
        if not result["OK"]:
            return result
        seDict = result["Value"]
        self.db.seDefinitions[seID]["SEDict"] = seDict
        if seDict:
            tmpDict = dict(seDict)
            tmpDict["FileName"] = ""
            result = pfnunparse(tmpDict)
            if result["OK"]:
                self.db.seDefinitions[seID]["SEDict"]["PFNPrefix"] = result["Value"]
        self.db.seDefinitions[seID]["LastUpdate"] = time.time()
        return S_OK(self.db.seDefinitions[seID])
Ejemplo n.º 38
0
    def __addDoubleSlash(self, res):
        """Utilities to add the double slash between the host(:port) and the path

        :param res: DIRAC return structure which contains an URL if S_OK
        :return: DIRAC structure with corrected URL
        """
        if not res["OK"]:
            return res
        url = res["Value"]
        res = pfnparse(url, srmSpecific=self.srmSpecificParse)
        if not res["OK"]:
            return res
        urlDict = res["Value"]
        urlDict["Path"] = "/" + urlDict["Path"]

        # Now, that's one heck of a disgusting hack
        # xrootd client is a bit faulty when managing
        # the connection cache, and ends up reusing an
        # existing connection for different users (security flaw...)
        # they have fixed it (to some extent starting from xrootd 4.10)
        # (https://github.com/xrootd/xrootd/issues/976)
        # BUT. They still can't consume properly the information when
        # the identity is passed in the url (root://url?gsiusrpxy=/tmp/myproxy)
        # So we apply a trick here which is to specify the proxy filename as a virtual user
        # This has no consequence (developer's own words), but to distinguish between users
        # Another ticket has been opened for that https://github.com/xrootd/xrootd/issues/992

        try:
            proxyLoc = getProxyLocation()
            if proxyLoc:
                # xroot does not support dots in the virtual user
                proxyLoc = os.path.basename(proxyLoc).replace(".", "")
                urlDict["Host"] = "%s@%s" % (proxyLoc, urlDict["Host"])
        except Exception as e:
            self.log.warn("Exception trying to add virtual user in the url",
                          repr(e))

        return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
Ejemplo n.º 39
0
    def test_pfnparse(self):
        castorPfn = '/castor/cern.ch/grid/lhcb/production/somefile.file'
        res = pfnparse(castorPfn)
        pfnDict = res['Value']
        res = pfnunparse(pfnDict)
        newPfn = res['Value']
        self.assertEqual(castorPfn, newPfn)

        rfioPfn = 'rfio:/castor/cern.ch/grid/lhcb/production/somefile.file'
        res = pfnparse(rfioPfn)
        pfnDict = res['Value']
        res = pfnunparse(pfnDict)
        newPfn = res['Value']
        self.assertEqual(rfioPfn, newPfn)

        miniSrmPfn = 'srm://srm.cern.ch/castor/cern.ch/grid/lhcb/prodution/somefile.file'
        res = pfnparse(miniSrmPfn)
        pfnDict = res['Value']
        res = pfnunparse(pfnDict)
        newPfn = res['Value']
        self.assertEqual(miniSrmPfn, newPfn)

        portSrmPfn = 'srm://srm.cern.ch:8443/castor/cern.ch/grid/lhcb/prodution/somefile.file'
        res = pfnparse(portSrmPfn)
        pfnDict = res['Value']
        res = pfnunparse(pfnDict)
        newPfn = res['Value']
        self.assertEqual(portSrmPfn, newPfn)

        srmPfn = 'srm://srm.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/grid/lhcb/prodution/somefile.file'
        res = pfnparse(srmPfn)
        pfnDict = res['Value']
        res = pfnunparse(pfnDict)
        newPfn = res['Value']
        self.assertEqual(srmPfn, newPfn)

        filePfn = 'file:/opt/dirac/somefile.file'
        res = pfnparse(filePfn)
        pfnDict = res['Value']
        res = pfnunparse(pfnDict)
        newPfn = res['Value']
        self.assertEqual(filePfn, newPfn)
Ejemplo n.º 40
0
  def test_pfnparse(self):
    castorPfn = '/castor/cern.ch/grid/lhcb/production/somefile.file'
    res = pfnparse(castorPfn)
    pfnDict = res['Value']
    res = pfnunparse(pfnDict)
    newPfn = res['Value']
    self.assertEqual(castorPfn,newPfn)

    rfioPfn = 'rfio:/castor/cern.ch/grid/lhcb/production/somefile.file'
    res = pfnparse(rfioPfn)
    pfnDict = res['Value']
    res = pfnunparse(pfnDict)
    newPfn = res['Value']
    self.assertEqual(rfioPfn,newPfn)

    miniSrmPfn = 'srm://srm.cern.ch/castor/cern.ch/grid/lhcb/prodution/somefile.file'
    res = pfnparse(miniSrmPfn)
    pfnDict = res['Value']
    res = pfnunparse(pfnDict)
    newPfn = res['Value']
    self.assertEqual(miniSrmPfn,newPfn)

    portSrmPfn = 'srm://srm.cern.ch:8443/castor/cern.ch/grid/lhcb/prodution/somefile.file'
    res = pfnparse(portSrmPfn)
    pfnDict = res['Value']
    res = pfnunparse(pfnDict)
    newPfn = res['Value']
    self.assertEqual(portSrmPfn,newPfn)

    srmPfn = 'srm://srm.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/grid/lhcb/prodution/somefile.file'
    res = pfnparse(srmPfn)
    pfnDict = res['Value']
    res = pfnunparse(pfnDict)
    newPfn = res['Value']
    self.assertEqual(srmPfn,newPfn)

    filePfn = 'file:/opt/dirac/somefile.file'
    res = pfnparse(filePfn)
    pfnDict = res['Value']
    res = pfnunparse(pfnDict)
    newPfn = res['Value']
    self.assertEqual(filePfn,newPfn)
Ejemplo n.º 41
0
 def getProtocolPfn( self, pfnDict, withPort ):
   """ From the pfn dict construct the SURL to be used
   """
   return pfnunparse( pfnDict )
Ejemplo n.º 42
0
 def getProtocolPfn(self, pfnDict, withPort):
     """ From the pfn dict construct the SURL to be used
 """
     return pfnunparse(pfnDict)
Ejemplo n.º 43
0
    def getSEDefinition(self, seID):
        """ Get the Storage Element definition
    """
        if isinstance(seID, str):
            result = self.getSEID(seID)
            if not result['OK']:
                return result
            seID = result['Value']

        if seID in self.db.seDefinitions:
            if (time.time() - self.db.seDefinitions[seID]['LastUpdate']
                ) < self.seUpdatePeriod:
                if self.db.seDefinitions[seID]['SEDict']:
                    return S_OK(self.db.seDefinitions[seID])
            se = self.db.seDefinitions[seID]['SEName']
        else:
            result = self.getSEName(seID)
            if not result['OK']:
                return result
            se = result['Value']
            self.db.seDefinitions[seID] = {}
            self.db.seDefinitions[seID]['SEName'] = se
            self.db.seDefinitions[seID]['SEDict'] = {}
            self.db.seDefinitions[seID]['LastUpdate'] = 0.

        # We have to refresh the SE definition from the CS
        result = gConfig.getSections('/Resources/StorageElements/%s' % se)
        if not result['OK']:
            return result
        pluginSection = result['Value'][0]
        result = gConfig.getOptionsDict('/Resources/StorageElements/%s/%s' %
                                        (se, pluginSection))
        if not result['OK']:
            return result
        seDict = result['Value']
        self.db.seDefinitions[seID]['SEDict'] = seDict
        # Get VO paths if any
        voPathDict = None
        result = gConfig.getOptionsDict(
            '/Resources/StorageElements/%s/%s/VOPath' % (se, pluginSection))
        if result['OK']:
            voPathDict = result['Value']
        if seDict:
            # A.T. Ports can be multiple, this can be better done using the Storage plugin
            # to provide the replica prefix to keep implementations in one place
            if 'Port' in seDict:
                ports = seDict['Port']
                if ',' in ports:
                    portList = [x.strip() for x in ports.split(',')]
                    random.shuffle(portList)
                    seDict['Port'] = portList[0]
            tmpDict = dict(seDict)
            tmpDict['FileName'] = ''
            result = pfnunparse(tmpDict)
            if result['OK']:
                self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result[
                    'Value']
            if voPathDict is not None:
                for vo in voPathDict:
                    tmpDict['Path'] = voPathDict[vo]
                    result = pfnunparse(tmpDict)
                    if result['OK']:
                        self.db.seDefinitions[seID]['SEDict'].setdefault(
                            "VOPrefix", {})[vo] = result['Value']
        self.db.seDefinitions[seID]['LastUpdate'] = time.time()
        return S_OK(self.db.seDefinitions[seID])