Exemplo n.º 1
0
  def setTasksDone( self, taskIDs ):
    """ This will update the status for a list of taskIDs to Done. """
    reqSelect = "SELECT * FROM Tasks WHERE TaskID IN (%s);" % intListToString( taskIDs )
    resSelect = self._query( reqSelect )
    if not resSelect['OK']:
      gLogger.error( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'setTasksDone', reqSelect, resSelect['Message'] ) )

    req = "UPDATE Tasks SET Status = 'Done', CompleteTime = UTC_TIMESTAMP() WHERE TaskID IN (%s);" % intListToString( taskIDs )
    res = self._update( req )
    if not res['OK']:
      gLogger.error( "StorageManagementDB.setTasksDone: Failed to set Tasks status to Done.", res['Message'] )
      return res

    for record in resSelect['Value']:
      gLogger.info( "%s.%s_DB: to_update Tasks =  %s" % ( self._caller(), 'setTasksDone', record ) )
      #fix, no individual queries
    reqSelect1 = "SELECT * FROM Tasks WHERE TaskID IN (%s);" % intListToString( taskIDs )
    resSelect1 = self._query( reqSelect1 )
    if not resSelect1['OK']:
      gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'setTasksDone', reqSelect1, resSelect1['Message'] ) )

    for record in resSelect1['Value']:
      gLogger.info( "%s.%s_DB: updated Tasks = %s" % ( self._caller(), 'setTasksDone', record ) )

    gLogger.debug( "StorageManagementDB.setTasksDone: Successfully updated %s Tasks with StageStatus=Done for taskIDs: %s." % ( res['Value'], taskIDs ) )
    return res
Exemplo n.º 2
0
  def _getDirectoryReplicas( self, dirID, allStatus=False, connection=False ):
    """ Get replicas for files in a given directory
    """
    replicaStatusIDs = []
    if not allStatus:
      for status in self.db.visibleReplicaStatus:
        result = self._getStatusInt( status, connection=connection )
        if result['OK']:
          replicaStatusIDs.append( result['Value'] )
    fileStatusIDs = []
    if not allStatus:
      for status in self.db.visibleFileStatus:
        result = self._getStatusInt( status, connection=connection )
        if result['OK']:
          fileStatusIDs.append( result['Value'] )

    if not self.db.lfnPfnConvention or self.db.lfnPfnConvention == "Weak":
      req = 'SELECT FF.FileName,FR.FileID,FR.SEID,FI.PFN FROM FC_Files as FF,'
      req += ' FC_Replicas as FR, FC_ReplicaInfo as FI'
      req += ' WHERE FF.FileID=FR.FileID AND FR.RepID=FI.RepID AND FF.DirID=%d ' % dirID
      if replicaStatusIDs:
        req += ' AND FR.Status in (%s)' % intListToString( replicaStatusIDs )
      if fileStatusIDs:
        req += ' AND FF.Status in (%s)' % intListToString( fileStatusIDs )
    else:
      req = "SELECT FF.FileName,FR.FileID,FR.SEID,'' FROM FC_Files as FF,"
      req += ' FC_Replicas as FR'
      req += ' WHERE FF.FileID=FR.FileID AND FF.DirID=%d ' % dirID
      if replicaStatusIDs:
        req += ' AND FR.Status in (%s)' % intListToString( replicaStatusIDs )
      if fileStatusIDs:
        req += ' AND FF.Status in (%s)' % intListToString( fileStatusIDs )

    result = self.db._query( req, connection )
    return result
Exemplo n.º 3
0
  def setStageComplete( self, replicaIDs ):
    # Daniela: FIX wrong PinExpiryTime (84000->86400 seconds = 1 day)

    reqSelect = "SELECT * FROM StageRequests WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    resSelect = self._query( reqSelect )
    if not resSelect['OK']:
      gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'setStageComplete', reqSelect, resSelect['Message'] ) )
      return resSelect

    req = "UPDATE StageRequests SET StageStatus='Staged',StageRequestCompletedTime = UTC_TIMESTAMP(),PinExpiryTime = DATE_ADD(UTC_TIMESTAMP(),INTERVAL ( PinLength / %s ) SECOND) WHERE ReplicaID IN (%s);" % ( THROTTLING_STEPS, intListToString( replicaIDs ) )
    res = self._update( req )
    if not res['OK']:
      gLogger.error( "StorageManagementDB.setStageComplete: Failed to set StageRequest completed.", res['Message'] )
      return res

    for record in resSelect['Value']:
      gLogger.info( "%s.%s_DB: to_update StageRequests =  %s" % ( self._caller(), 'setStageComplete', record ) )

    reqSelect1 = "SELECT * FROM StageRequests WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    resSelect1 = self._query( reqSelect1 )
    if not resSelect1['OK']:
      gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'setStageComplete', reqSelect1, resSelect1['Message'] ) )

    for record in resSelect1['Value']:
      gLogger.info( "%s.%s_DB: updated StageRequests = %s" % ( self._caller(), 'setStageComplete', record ) )

    gLogger.debug( "StorageManagementDB.setStageComplete: Successfully updated %s StageRequests table with StageStatus=Staged for ReplicaIDs: %s." % ( res['Value'], replicaIDs ) )
    return res
Exemplo n.º 4
0
  def __findFilesForSE( self, se, dirList ):
    """ Find files in the given list of directories having replicas in the given se(s)
    """
    seList = se
    if type( se ) in types.StringTypes:
      seList = [se]
    seIDs = []
    for se in seList:
      result = self.db.seManager.getSEID( se )
      if not result['OK']:
        return result
      seIDs.append( result['Value'] )
    seString = intListToString( seIDs )
    dirString = intListToString( dirList )

    req = "SELECT F.FileID FROM FC_Files as F, FC_Replicas as R WHERE F.DirID IN (%s)" % dirString
    req += " AND R.SEID IN (%s) AND F.FileID=R.FileID" % seString
    result = self.db._query( req )
    if not result['OK']:
      return result
    if not result['Value']:
      return S_OK( [] )

    fileList = []
    for row in result['Value']:
      fileID = row[0]
      fileList.append( fileID )

    return S_OK( fileList )
Exemplo n.º 5
0
 def _getFileReplicas(self, fileIDs, fields=["PFN"], connection=False):
     connection = self._getConnection(connection)
     if not fileIDs:
         return S_ERROR("No such file or directory")
     req = "SELECT FileID,SEID,Status,%s FROM FC_Replicas WHERE FileID IN (%s);" % (
         intListToString(fields),
         intListToString(fileIDs),
     )
     res = self.db._query(req, connection)
     if not res["OK"]:
         return res
     replicas = {}
     for tuple in res["Value"]:
         fileID = tuple[0]
         if not replicas.has_key(fileID):
             replicas[fileID] = {}
         seID = tuple[1]
         res = self.db.seManager.getSEName(seID)
         if not res["OK"]:
             continue
         seName = res["Value"]
         statusID = tuple[2]
         res = self._getIntStatus(statusID, connection=connection)
         if not res["OK"]:
             continue
         status = res["Value"]
         replicas[fileID][seName] = {"Status": status}
         replicas[fileID][seName].update(dict(zip(fields, tuple[3:])))
     for fileID in fileIDs:
         if not replicas.has_key(fileID):
             replicas[fileID] = {}
     return S_OK(replicas)
Exemplo n.º 6
0
    def _getFileReplicas(self, fileIDs, fields_input=["PFN"], allStatus=False, connection=False):
        """ Get replicas for the given list of files specified by their fileIDs
    """
        fields = list(fields_input)
        connection = self._getConnection(connection)
        res = self.__getFileIDReplicas(fileIDs, allStatus=allStatus, connection=connection)
        if not res["OK"]:
            return res
        fileIDDict = res["Value"]
        if fileIDDict:
            if "Status" in fields:
                fields.remove("Status")
            repIDDict = {}
            if fields:
                req = "SELECT RepID,%s FROM FC_ReplicaInfo WHERE RepID IN (%s);" % (
                    intListToString(fields),
                    intListToString(fileIDDict.keys()),
                )
                res = self.db._query(req, connection)
                if not res["OK"]:
                    return res
                for tuple_ in res["Value"]:
                    repID = tuple_[0]
                    repIDDict[repID] = dict(zip(fields, tuple_[1:]))
                    statusID = fileIDDict[repID][2]
                    res = self._getIntStatus(statusID, connection=connection)
                    if not res["OK"]:
                        continue
                    repIDDict[repID]["Status"] = res["Value"]
            else:
                for repID in fileIDDict:
                    statusID = fileIDDict[repID][2]
                    res = self._getIntStatus(statusID, connection=connection)
                    if not res["OK"]:
                        continue
                    repIDDict[repID] = {"Status": res["Value"]}
        seDict = {}
        replicas = {}
        for repID in fileIDDict.keys():
            fileID, seID, statusID = fileIDDict[repID]
            replicas.setdefault(fileID, {})
            if not seID in seDict:
                res = self.db.seManager.getSEName(seID)
                if not res["OK"]:
                    continue
                seDict[seID] = res["Value"]
            seName = seDict[seID]
            replicas[fileID][seName] = repIDDict.get(repID, {})

        if len(replicas) != len(fileIDs):
            for fileID in fileIDs:
                if not replicas.has_key(fileID):
                    replicas[fileID] = {}

        return S_OK(replicas)
Exemplo n.º 7
0
 def _getFileDescendents( self, fileIDs, depths, connection = False ):
   connection = self._getConnection( connection )
   req = "SELECT AncestorID, FileID, AncestorDepth FROM FC_FileAncestors WHERE AncestorID IN (%s)" \
                               % intListToString( fileIDs )
   if depths:
     req = "%s AND AncestorDepth IN (%s);" % ( req, intListToString( depths ) )
   res = self.db._query( req, connection )
   if not res['OK']:
     return res
   fileIDAncestors = {}
   for ancestorID, fileID, depth in res['Value']:
     if not fileIDAncestors.has_key( ancestorID ):
       fileIDAncestors[ancestorID] = {}
     fileIDAncestors[ancestorID][fileID] = depth
   return S_OK( fileIDAncestors )
Exemplo n.º 8
0
 def _getFileAncestors(self, fileIDs, depths=[], connection=False):
   connection = self._getConnection(connection)
   req = "SELECT FileID, AncestorID, AncestorDepth FROM FC_FileAncestors WHERE FileID IN (%s)" \
       % intListToString(fileIDs)
   if depths:
     req = "%s AND AncestorDepth IN (%s);" % (req, intListToString(depths))
   res = self.db._query(req, connection)
   if not res['OK']:
     return res
   fileIDAncestors = {}
   for fileID, ancestorID, depth in res['Value']:
     if fileID not in fileIDAncestors:
       fileIDAncestors[fileID] = {}
     fileIDAncestors[fileID][ancestorID] = depth
   return S_OK(fileIDAncestors)
Exemplo n.º 9
0
  def _getFileMetadataByID( self, fileIDs, connection=False ):
    """ Get standard file metadata for a list of files specified by FileID
        :param fileIDS : list of file Ids

        :returns S_OK(files), where files is a dictionary indexed on fileID
                            and the values dictionaries containing the following info:
                            ["FileID", "Size", "UID", "GID", "s.Status", "GUID", "CreationDate"]
    """

    # Format the filenames and status to be used in a IN clause in the sotred procedure
    formatedFileIds = intListToString( fileIDs )
    result = self.db.executeStoredProcedureWithCursor( 'ps_get_all_info_for_file_ids', ( formatedFileIds, ) )
    if not result['OK']:
      return result

    rows = result['Value']

    fieldNames = ["FileID", "Size", "UID", "GID", "s.Status", "GUID", "CreationDate"]

    resultDict = {}

    for row in rows:
      rowDict = dict( zip( fieldNames, row ) )
      rowDict["Size"] = int( rowDict["Size"] )
      rowDict["UID"] = int( rowDict["UID"] )
      rowDict["GID"] = int( rowDict["GID"] )
      resultDict[rowDict["FileID"]] = rowDict

    return S_OK( resultDict )
Exemplo n.º 10
0
    def _getFileLFNs_old(self, fileIDs):
        """ Get the file LFNs for a given list of file IDs
    """

        start = time.time()

        stringIDs = intListToString(fileIDs)
        req = "SELECT DirID, FileID, FileName from FC_Files WHERE FileID IN ( %s )" % stringIDs
        result = self.db._query(req)
        if not result["OK"]:
            return result

        dirPathDict = {}
        fileNameDict = {}
        for row in result["Value"]:
            if not row[0] in dirPathDict:
                dirPathDict[row[0]] = self.db.dtree.getDirectoryPath(row[0])["Value"]
            fileNameDict[row[1]] = "%s/%s" % (dirPathDict[row[0]], row[2])

        failed = {}
        successful = fileNameDict
        for id in fileIDs:
            if not id in fileNameDict:
                failed[id] = "File ID not found"

        return S_OK({"Successful": successful, "Failed": failed})
Exemplo n.º 11
0
  def updateReplicaInformation( self, replicaTuples ):
    """ This method set the replica size information and pfn for the requested storage element.  """
    for replicaID, pfn, size in replicaTuples:
      reqSelect = "SELECT * FROM CacheReplicas WHERE ReplicaID = %s and Status != 'Cancelled';" % ( replicaID )
      resSelect = self._query( reqSelect )
      if not resSelect['OK']:
        gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'updateReplicaInformation', reqSelect, resSelect['Message'] ) )

      req = "UPDATE CacheReplicas SET PFN = '%s', Size = %s, Status = 'Waiting' WHERE ReplicaID = %s and Status != 'Cancelled';" % ( pfn, size, replicaID )
      res = self._update( req )
      if not res['OK']:
        gLogger.error( 'StagerDB.updateReplicaInformation: Failed to insert replica information.', res['Message'] )

      replicaIDs = []
      for record in resSelect['Value']:
        replicaIDs.append( record[0] )
        gLogger.info( "%s.%s_DB: to_update CacheReplicas =  %s" % ( self._caller(), 'updateReplicaInformation', record ) )

      reqSelect1 = "SELECT * FROM CacheReplicas WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
      resSelect1 = self._query( reqSelect1 )
      if not resSelect1['OK']:
        gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'updateReplicaInformation', reqSelect1, resSelect1['Message'] ) )

      for record in resSelect1['Value']:
        gLogger.info( "%s.%s_DB: updated CacheReplicas = %s" % ( self._caller(), 'updateReplicaInformation', record ) )

      gLogger.debug( 'StagerDB.updateReplicaInformation: Successfully updated CacheReplicas record With Status=Waiting, for ReplicaID= %s' % ( replicaID ) )
    return S_OK()
Exemplo n.º 12
0
  def updateReplicaFailure( self, terminalReplicaIDs ):
    """ This method sets the status to Failure with the failure reason for the supplied Replicas. """
    res = self.updateReplicaStatus( terminalReplicaIDs.keys(), 'Failed' )
    if not res['OK']:
      return res
    updated = res['Value']
    if not updated:
      return S_OK( updated )
    for replicaID in updated:
      reqSelect = "Select * FROM CacheReplicas WHERE ReplicaID = %d" % ( replicaID )
      resSelect = self._query( reqSelect )
      if not resSelect['OK']:
        gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'updateReplicaFailure', reqSelect, resSelect['Message'] ) )

      req = "UPDATE CacheReplicas SET Reason = '%s' WHERE ReplicaID = %d" % ( terminalReplicaIDs[replicaID], replicaID )
      res = self._update( req )
      if not res['OK']:
        gLogger.error( 'StorageManagementDB.updateReplicaFailure: Failed to update replica fail reason.', res['Message'] )
        return res

      replicaIDs = []
      for record in resSelect['Value']:
        replicaIDs.append( record[0] )
        gLogger.info( "%s.%s_DB: to_update CacheReplicas =  %s" % ( self._caller(), 'updateReplicaFailure', record ) )

      reqSelect1 = "SELECT * FROM CacheReplicas WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
      resSelect1 = self._query( reqSelect1 )
      if not resSelect1['OK']:
        gLogger.info( "%s.%s_DB: problem retrieving records: %s. %s" % ( self._caller(), 'updateReplicaFailure', reqSelect1, resSelect1['Message'] ) )

      for record in resSelect1['Value']:
        gLogger.info( "%s.%s_DB: updated CacheReplicas = %s" % ( self._caller(), 'updateReplicaFailure', record ) )

    return S_OK( updated )
Exemplo n.º 13
0
 def getCacheReplicas( self, condDict = {}, older = None, newer = None, timeStamp = 'LastUpdate', orderAttribute = None, limit = None, connection = False ):
   """ Get cache replicas for the supplied selection with support for the web standard structure """
   connection = self.__getConnection( connection )
   req = "SELECT %s FROM CacheReplicas" % ( intListToString( self.REPLICAPARAMS ) )
   originalFileIDs = {}
   if condDict or older or newer:
     if condDict.has_key( 'TaskID' ):
       taskIDs = condDict.pop( 'TaskID' )
       if type( taskIDs ) not in ( types.ListType, types.TupleType ):
         taskIDs = [taskIDs]
       res = self._getTaskReplicaIDs( taskIDs, connection = connection )
       if not res['OK']:
         return res
       condDict['ReplicaID'] = res['Value']
     req = "%s %s" % ( req, self.buildCondition( condDict, older, newer, timeStamp, orderAttribute, limit ) )
   res = self._query( req, connection )
   if not res['OK']:
     return res
   cacheReplicas = res['Value']
   resultDict = {}
   for row in cacheReplicas:
     resultDict[row[0]] = dict( zip( self.REPLICAPARAMS[1:], row[1:] ) )
   result = S_OK( resultDict )
   result['Records'] = cacheReplicas
   result['ParameterNames'] = self.REPLICAPARAMS
   return result
Exemplo n.º 14
0
  def _getReplicaTasks( self, replicaIDs, connection = False ):
    connection = self.__getConnection( connection )
    #Daniela: should select only tasks with complete replicaID sets
    #only if ALL Replicas belonging to a Task have a certain state, the task should be selected for state update
    req = "SELECT DISTINCT(TaskID) FROM TaskReplicas WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    res = self._query( req, connection )
    if not res['OK']:
      return res
    taskIDs = [row[0] for row in res['Value']]

    # fix
    finalTaskIDs = []
    for taskID in taskIDs:
      subreq = "SELECT ReplicaID FROM CacheReplicas WHERE Status = (SELECT Status FROM Tasks WHERE TaskID = %s);" % taskID
      subres = self._query( subreq, connection )
      if not subres['OK']:
        return subres
      replicaIDsForTask = [row[0] for row in subres['Value']]
      setOriginalReplicaIDs = Set( replicaIDs )
      setReplicaIDsForTask = Set( replicaIDsForTask )
      if setReplicaIDsForTask <= setOriginalReplicaIDs:
        finalTaskIDs.append( taskID )
    #end_fix

    return S_OK( finalTaskIDs )
Exemplo n.º 15
0
    def _getFileLFNs(self, fileIDs):
        """ Get the file LFNs for a given list of file IDs
    """

        start = time.time()

        stringIDs = intListToString(fileIDs)
        treeTable = self.db.dtree.getTreeTable()

        req = (
            "SELECT F.FileID, CONCAT(D.DirName,'/',F.FileName) from FC_Files as F, %s as D WHERE F.FileID IN ( %s ) AND F.DirID=D.DirID"
            % (treeTable, stringIDs)
        )
        result = self.db._query(req)
        if not result["OK"]:
            return result

        fileNameDict = {}
        for row in result["Value"]:
            fileNameDict[row[0]] = row[1]

        failed = {}
        successful = fileNameDict
        if len(fileNameDict) != len(fileIDs):
            for id in fileIDs:
                if not id in fileNameDict:
                    failed[id] = "File ID not found"

        return S_OK({"Successful": successful, "Failed": failed})
Exemplo n.º 16
0
  def __deleteFileReplicas( self, fileIDs, connection = False ):
    """ Delete all the replicas from the file ids
        :param fileIDs list of file ids

        :returns S_OK() or S_ERROR(msg)
    """

    connection = self._getConnection(connection)

    if not fileIDs:
      return S_OK()

    formatedFileIds = intListToString( fileIDs )



    result = self.db.executeStoredProcedureWithCursor( 'ps_delete_replicas_from_file_ids', ( formatedFileIds, ) )
    if not result['OK']:
      return result

    errno, msg = result['Value'][0]

    if errno:
      return S_ERROR( msg )

    return S_OK()
Exemplo n.º 17
0
  def _getFileLFNs(self, fileIDs):
    """ Get the file LFNs for a given list of file IDs
    """
    stringIDs = intListToString(fileIDs)
    treeTable = self.db.dtree.getTreeTable()

    req = "SELECT F.FileID, CONCAT(D.DirName,'/',F.FileName) from FC_Files as F,\
        %s as D WHERE F.FileID IN ( %s ) AND F.DirID=D.DirID" % (
        treeTable, stringIDs)
    result = self.db._query(req)
    if not result['OK']:
      return result

    fileNameDict = {}
    for row in result['Value']:
      fileNameDict[row[0]] = row[1]

    failed = {}
    successful = fileNameDict
    if len(fileNameDict) != len(fileIDs):
      for id_ in fileIDs:
        if id_ not in fileNameDict:
          failed[id_] = "File ID not found"

    return S_OK({'Successful': successful, 'Failed': failed})
Exemplo n.º 18
0
 def getStageRequests( self, condDict = {}, older = None, newer = None, timeStamp = 'StageRequestSubmitTime', orderAttribute = None, limit = None, connection = False ):
   """ Get stage requests for the supplied selection with support for web standard structure """
   connection = self.__getConnection( connection )
   req = "SELECT %s FROM StageRequests" % ( intListToString( self.STAGEPARAMS ) )
   if condDict or older or newer:
     if condDict.has_key( 'TaskID' ):
       taskIDs = condDict.pop( 'TaskID' )
       if type( taskIDs ) not in ( types.ListType, types.TupleType ):
         taskIDs = [taskIDs]
       res = self._getTaskReplicaIDs( taskIDs, connection = connection )
       if not res['OK']:
         return res
       if res['Value']:
         condDict['ReplicaID'] = res['Value']
       else:
         condDict['ReplicaID'] = [-1]
     req = "%s %s" % ( req, self.buildCondition( condDict, older, newer, timeStamp, orderAttribute, limit ) )
   res = self._query( req, connection )
   if not res['OK']:
     return res
   stageRequests = res['Value']
   resultDict = {}
   for row in stageRequests:
     resultDict[row[0]] = dict( zip( self.STAGEPARAMS[1:], row[1:] ) )
   result = S_OK( resultDict )
   result['Records'] = stageRequests
   result['ParameterNames'] = self.STAGEPARAMS
   return result
Exemplo n.º 19
0
  def removeUnlinkedReplicas( self, connection = False ):
    """ This will remove from the CacheReplicas tables where there are no associated links. """
    connection = self.__getConnection( connection )
    req = "SELECT ReplicaID from CacheReplicas WHERE Links = 0;"
    res = self._query( req, connection )
    if not res['OK']:
      gLogger.error( "StorageManagementDB.removeUnlinkedReplicas. Problem selecting entries from CacheReplicas where Links = 0." )
      return res
    replicaIDs = []
    for tuple in res['Value']:
      replicaIDs.append( tuple[0] )
    if not replicaIDs:
      return S_OK()


    reqSelect = "SELECT * FROM StageRequests WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    resSelect = self._query( reqSelect )
    if not resSelect['OK']:
      gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'removeUnlinkedReplicas', reqSelect, resSelect['Message'] ) )
    for record in resSelect['Value']:
      gLogger.info( "%s.%s_DB: to_delete StageRequests = %s" % ( self._caller(), 'removeUnlinkedReplicas', record ) )

    req = "DELETE FROM StageRequests WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    res = self._update( req, connection )
    if not res['OK']:
      gLogger.error( "StorageManagementDB.removeUnlinkedReplicas. Problem deleting from StageRequests." )
      return res
    gLogger.info( "%s.%s_DB: deleted StageRequests" % ( self._caller(), 'removeUnlinkedReplicas' ) )

    gLogger.debug( "StorageManagementDB.removeUnlinkedReplicas: Successfully removed %s StageRequests entries for ReplicaIDs: %s." % ( res['Value'], replicaIDs ) )

    reqSelect = "SELECT * FROM CacheReplicas WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    resSelect = self._query( reqSelect )
    if not resSelect['OK']:
      gLogger.info( "%s.%s_DB: problem retrieving record: %s. %s" % ( self._caller(), 'removeUnlinkedReplicas', reqSelect, resSelect['Message'] ) )
    for record in resSelect['Value']:
      gLogger.info( "%s.%s_DB: to_delete CacheReplicas =  %s" % ( self._caller(), 'removeUnlinkedReplicas', record ) )

    req = "DELETE FROM CacheReplicas WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    res = self._update( req, connection )
    if res['OK']:
      gLogger.info( "%s.%s_DB: deleted CacheReplicas" % ( self._caller(), 'removeUnlinkedReplicas' ) )
      gLogger.debug( "StorageManagementDB.removeUnlinkedReplicas: Successfully removed %s CacheReplicas entries for ReplicaIDs: %s." % ( res['Value'], replicaIDs ) )
    else:
      gLogger.error( "StorageManagementDB.removeUnlinkedReplicas. Problem removing entries from CacheReplicas." )
    return res
Exemplo n.º 20
0
 def __getFileIDReplicas(self,fileIDs,allStatus=False,connection=False):
   connection = self._getConnection(connection)
   if not fileIDs:
     return S_ERROR("No such file or directory")
   req = "SELECT FileID,SEID,RepID,Status FROM FC_Replicas WHERE FileID IN (%s)" % (intListToString(fileIDs))
   if not allStatus:
     statusIDs = []
     for status in self.db.visibleReplicaStatus:
       result = self._getStatusInt( status, connection=connection )
       if result['OK']:
         statusIDs.append( result['Value'] )
     req += " AND Status in (%s)" % (intListToString(statusIDs))
   res = self.db._query(req,connection)
   if not res['OK']:
     return res
   fileIDDict = {}
   for fileID,seID,repID,statusID in res['Value']:
     fileIDDict[repID] = ( fileID, seID, statusID )
   return S_OK(fileIDDict)
Exemplo n.º 21
0
  def cleanUpFTSFiles( self, requestID, fileIDs ):
    """ delete FTSFiles for given :requestID: and list of :fileIDs:

    :param int requestID: ReqDB.Request.RequestID
    :param list fileIDs: [ ReqDB.File.FileID, ... ]
    """
    query = "DELETE FROM `FTSFile` WHERE `RequestID`= %s and `FileID` IN (%s)" % ( requestID,
                                                                                   intListToString( fileIDs ) )
    deleteFiles = self._transaction( [query] )
    return deleteFiles
Exemplo n.º 22
0
 def _setFileParameter(self, fileID, paramName, paramValue, connection=False):
     connection = self._getConnection(connection)
     if type(fileID) not in [TupleType, ListType]:
         fileID = [fileID]
     req = "UPDATE FC_Files SET %s='%s', ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (%s)" % (
         paramName,
         paramValue,
         intListToString(fileID),
     )
     return self.db._update(req, connection)
Exemplo n.º 23
0
  def _getReplicaIDTasks( self, replicaIDs, connection = False ):
    if not replicaIDs:
      return S_OK( [] )
    req = "SELECT DISTINCT(TaskID) FROM TaskReplicas WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
    res = self._query( req, connection )
    if not res['OK']:
      return res
    taskIDs = [row[0] for row in res['Value']]

    return S_OK( taskIDs )
Exemplo n.º 24
0
 def _insertFileAncestors( self, fileID, ancestorDict, connection = False ):
   connection = self._getConnection( connection )
   ancestorTuples = []
   for ancestorID, depth in ancestorDict.items():
     ancestorTuples.append( "(%d,%d,%d)" % ( fileID, ancestorID, depth ) )
   if not ancestorTuples:
     return S_OK()
   req = "INSERT INTO FC_FileAncestors (FileID, AncestorID, AncestorDepth) VALUES %s" \
                             % intListToString( ancestorTuples )
   return self.db._update( req, connection )
Exemplo n.º 25
0
 def _getReplicaIDTasks( self, replicaIDs, connection = False ):
   req = "SELECT TaskID FROM TaskReplicas WHERE ReplicaID IN (%s);" % intListToString( replicaIDs )
   res = self._query( req, connection )
   if not res['OK']:
     return res
   taskIDs = []
   for tuple in res['Value']:
     taskID = tuple[0]
     if not taskID in taskIDs:
       taskIDs.append( taskID )
   return S_OK( taskIDs )
Exemplo n.º 26
0
 def getRequestForJobs( self, jobIDs ):
   """ Get the request names associated to the jobsIDs
   """
   req = "SELECT JobID,RequestName from Requests where JobID IN (%s);" % intListToString( jobIDs )
   res = self._query( req )
   if not res:
     return res
   jobIDs = {}
   for jobID, requestName in res['Value']:
     jobIDs[jobID] = requestName
   return S_OK( jobIDs )
Exemplo n.º 27
0
  def deleteFTSFiles( self, operationID, opFileIDList = None ):
    """ delete FTSFiles for reschedule

    :param int operationID: ReqDB.Operation.OperationID
    :param list opFileIDList: [ ReqDB.File.FileID, ... ]
    """
    query = [ "DELETE FROM `FTSFile` WHERE OperationID = %s" % operationID ]
    if opFileIDList:
      query.append( " AND `FileID` IN (%s)" % intListToString( opFileIDList ) )
    query.append( ";" )
    return self._update( "".join( query ) )
Exemplo n.º 28
0
 def __findDirs(self,paths,metadata=['DirName']):
   dirs = {}
   req = "SELECT DirID,%s FROM DirectoryInfo WHERE DirName IN (%s)" % (intListToString(metadata),stringListToString(paths))
   res = self.db._query(req)
   if not res['OK']:
     return res
   if not res['Value']:
     return S_OK(dirs)
   for tuple in res['Value']:
     dirID = tuple[0]
     dirs[dirID] = dict(zip(metadata,tuple[1:]))
   return S_OK(dirs)
Exemplo n.º 29
0
 def __deleteReplicas(self, replicaTuples, connection=False):
     connection = self._getConnection(connection)
     deleteTuples = []
     for fileID, seID in replicaTuples:
         if type(seID) in StringTypes:
             res = self.db.seManager.findSE(seID)
             if not res["OK"]:
                 return res
             seID = res["Value"]
         deleteTuples.append("(%d,%d)" % (fileID, seID))
     req = "DELETE FROM FC_Replicas WHERE (FileID,SEID) IN (%s)" % intListToString(deleteTuples)
     return self.db._update(req, connection)
Exemplo n.º 30
0
 def _getDirectoryFiles(self, dirID, fileNames, metadata, allStatus=False, connection=False):
     connection = self._getConnection(connection)
     # metadata can be any of ['FileID','Size','UID','GID','Checksum','ChecksumType','Type','CreationDate','ModificationDate','Mode','Status']
     req = "SELECT FileName,%s FROM FC_Files WHERE DirID=%d" % (intListToString(metadata), dirID)
     if not allStatus:
         statusIDs = []
         res = self._getStatusInt("AprioriGood", connection=connection)
         if res["OK"]:
             statusIDs.append(res["Value"])
         if statusIDs:
             req = "%s AND Status IN (%s)" % (req, intListToString(statusIDs))
     if fileNames:
         req = "%s AND FileName IN (%s)" % (req, stringListToString(fileNames))
     res = self.db._query(req, connection)
     if not res["OK"]:
         return res
     files = {}
     for tuple in res["Value"]:
         fileName = tuple[0]
         files[fileName] = dict(zip(metadata, tuple[1:]))
     return S_OK(files)
Exemplo n.º 31
0
  def getDirectoryPaths( self, dirIDList ):
    """ Get directory names by directory ID list

        :param dirIDList: list of dirIds
        :returns: S_OK( { dirID : dirName} )
    """

    dirs = dirIDList
    if not isinstance( dirIDList, list ):
      dirs = [dirIDList]


    dirDict = {}

    # Format the list
    dIds = intListToString( dirs )
    result = self.db.executeStoredProcedureWithCursor( 'ps_get_dirNames_from_ids', ( dIds, ) )
    if not result['OK']:
      return result

    for dirId, dirName in result['Value']:
      dirDict[dirId] = dirName

    return S_OK( dirDict )
Exemplo n.º 32
0
    def _getFileLFNs(self, fileIDs):
        """ Get the file LFNs for a given list of file IDs
    """

        stringIDs = intListToString(fileIDs)
        req = "SELECT DirID, FileID, FileName from FC_Files WHERE FileID IN ( %s )" % stringIDs
        result = self.db._query(req)
        if not result['OK']:
            return result
        dirPathDict = {}
        fileNameDict = {}
        for row in result['Value']:
            if not row[0] in dirPathDict:
                dirPathDict[row[0]] = self.db.dtree.getDirectoryPath(
                    row[0])['Value']
            fileNameDict[row[1]] = '%s/%s' % (dirPathDict[row[0]], row[2])

        failed = {}
        successful = fileNameDict
        for id in fileIDs:
            if not id in fileNameDict:
                failed[id] = "File ID not found"

        return S_OK({'Successful': successful, 'Failed': failed})
Exemplo n.º 33
0
    def __findFilesByMetadata(self, metaDict, dirList, credDict):
        """ Find a list of file IDs meeting the metaDict requirements and belonging
        to directories in dirList 
    """

        # 1.- classify Metadata keys
        storageElement = None
        standardMetaDict = {}
        userMetaDict = {}
        for meta, value in metaDict.items():
            if meta == "SE":
                storageElement = value
            elif meta in FILE_STANDARD_METAKEYS:
                standardMetaDict[meta] = value
            else:
                userMetaDict[meta] = value

        tablesAndConditions = []
        # 2.- standard search
        result = self.__buildStandardMetaQuery(standardMetaDict)
        if not result['OK']:
            return result
        tablesAndConditions.extend(result['Value'])
        # 3.- user search
        result = self.__buildUserMetaQuery(userMetaDict)
        if not result['OK']:
            return result
        tablesAndConditions.extend(result['Value'])
        # 4.- SE constrain
        result = self.__buildSEQuery(storageElement)
        if not result['OK']:
            return result
        tablesAndConditions.extend(result['Value'])

        query = 'SELECT F.FileID FROM '

        conditions = []
        tables = ['FC_Files as F']

        if dirList:
            dirString = intListToString(dirList)
            conditions.append("F.DirID in (%s)" % dirString)

        counter = 0
        for table, condition in tablesAndConditions:
            counter += 1
            tables.append('%s as M%d' % (table, counter))
            table = 'M%d' % counter
            condition = condition % table + ' AND F.FileID = %s.FileID' % table
            conditions.append('( %s )' % condition)

        query += ', '.join(tables)
        if conditions:
            query += ' WHERE %s' % ' AND '.join(conditions)

        result = self.db._query(query)
        if not result['OK']:
            return result
        if not result['Value']:
            return S_OK([])


#     fileList = [ row[0] for row in result['Value' ] ]
        fileList = []
        for row in result['Value']:
            fileID = row[0]
            fileList.append(fileID)

        return S_OK(fileList)
Exemplo n.º 34
0
    def __findFilesByMetadata(self, metaDict, dirList, credDict):
        """ Find a list of file IDs meeting the metaDict requirements and belonging
        to directories in dirList
    """
        # 1.- classify Metadata keys
        storageElements = None
        standardMetaDict = {}
        userMetaDict = {}
        leftJoinTables = []
        for meta, value in metaDict.items():
            if meta == "SE":
                if isinstance(value, DictType):
                    storageElements = value.get('in', [])
                else:
                    storageElements = [value]
            elif meta in FILE_STANDARD_METAKEYS:
                standardMetaDict[meta] = value
            else:
                userMetaDict[meta] = value

        tablesAndConditions = []
        leftJoinTables = []
        # 2.- standard search
        if standardMetaDict:
            result = self.__buildStandardMetaQuery(standardMetaDict)
            if not result['OK']:
                return result
            tablesAndConditions.extend(result['Value'])
        # 3.- user search
        if userMetaDict:
            result = self.__buildUserMetaQuery(userMetaDict)
            if not result['OK']:
                return result
            tablesAndConditions.extend(result['Value'])
            leftJoinTables = result['LeftJoinTables']
        # 4.- SE constraint
        if storageElements:
            result = self.__buildSEQuery(storageElements)
            if not result['OK']:
                return result
            tablesAndConditions.extend(result['Value'])

        query = 'SELECT F.FileID FROM FC_Files F '
        conditions = []
        tables = []

        if dirList:
            dirString = intListToString(dirList)
            conditions.append("F.DirID in (%s)" % dirString)

        counter = 0
        for table, condition in tablesAndConditions:
            if table == 'FC_FileInfo':
                query += 'INNER JOIN FC_FileInfo FI USING( FileID ) '
                condition = condition.replace('%%', '%')
            elif table == 'FC_Files':
                condition = condition.replace('%%', '%')
            else:
                counter += 1
                if table in leftJoinTables:
                    tables.append('LEFT JOIN %s M%d USING( FileID )' %
                                  (table, counter))
                else:
                    tables.append('INNER JOIN %s M%d USING( FileID )' %
                                  (table, counter))
                table = 'M%d' % counter
                condition = condition % table
            conditions.append(condition)

        query += ' '.join(tables)
        if conditions:
            query += ' WHERE %s' % ' AND '.join(conditions)

        result = self.db._query(query)
        if not result['OK']:
            return result
        if not result['Value']:
            return S_OK([])


#     fileList = [ row[0] for row in result['Value' ] ]
        fileList = []
        for row in result['Value']:
            fileID = row[0]
            fileList.append(fileID)

        return S_OK(fileList)
Exemplo n.º 35
0
 def __deleteFiles(self, fileIDs, connection=False):
   connection = self._getConnection(connection)
   if not fileIDs:
     return S_OK()
   req = "DELETE FROM FC_Files WHERE FileID in (%s)" % (intListToString(fileIDs))
   return self.db._update(req, connection)
Exemplo n.º 36
0
 def __getFileIDReplicas(self,fileIDs,connection=False):
   connection = self._getConnection(connection)
   if not fileIDs:
     return S_ERROR("No such file or directory")
   req = "SELECT FileID,SEID,RepID,Status FROM FC_Replicas WHERE FileID IN (%s);" % (intListToString(fileIDs))
   res = self.db._query(req,connection)
   if not res['OK']:
     return res
   fileIDDict = {}
   for fileID,seID,repID,status in res['Value']:
     fileIDDict[repID] = {'FileID':fileID,'SEID':seID,'Status':status}
   return S_OK(fileIDDict)
Exemplo n.º 37
0
 def _setFileParameter( self, fileID, paramName, paramValue, connection = False ):
   connection = self._getConnection(connection)
   if type(fileID) not in [TupleType,ListType]:
     fileID = [fileID]
     
   if paramName in ['UID','GID','Status','Size']:
     # Treat primary file attributes specially
     req = "UPDATE FC_Files SET %s='%s' WHERE FileID IN (%s)" % ( paramName, paramValue, intListToString( fileID ) )
     result = self.db._update(req,connection)
     if not result['OK']:
       return result
     req = "UPDATE FC_FileInfo SET ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (%s)" % intListToString( fileID )
   else:  
     req = "UPDATE FC_FileInfo SET %s='%s', ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (%s)" % ( paramName, paramValue,
                                                                                                      intListToString( fileID ) )
   return self.db._update(req,connection)
Exemplo n.º 38
0
    def __findFilesByMetadata(self, metaDict, dirList, credDict):
        """Find a list of file IDs meeting the metaDict requirements and belonging
        to directories in dirList

        :param dict metaDict: dictionary with the file metadata
        :param list dirList: list of directories to look into

        :return: S_OK/S_ERROR, Value - list of IDs of found files
        """
        # 1.- classify Metadata keys
        storageElements = None
        standardMetaDict = {}
        userMetaDict = {}
        leftJoinTables = []
        for meta, value in metaDict.items():
            if meta == "SE":
                if isinstance(value, dict):
                    storageElements = value.get("in", [])
                else:
                    storageElements = [value]
            elif meta in FILE_STANDARD_METAKEYS:
                standardMetaDict[meta] = value
            else:
                userMetaDict[meta] = value

        tablesAndConditions = []
        leftJoinTables = []
        # 2.- standard search
        if standardMetaDict:
            result = self.__buildStandardMetaQuery(standardMetaDict)
            if not result["OK"]:
                return result
            tablesAndConditions.extend(result["Value"])
        # 3.- user search
        if userMetaDict:
            result = self.__buildUserMetaQuery(userMetaDict)
            if not result["OK"]:
                return result
            tablesAndConditions.extend(result["Value"])
            leftJoinTables = result["LeftJoinTables"]
        # 4.- SE constraint
        if storageElements:
            result = self.__buildSEQuery(storageElements)
            if not result["OK"]:
                return result
            tablesAndConditions.extend(result["Value"])

        query = "SELECT F.FileID FROM FC_Files F "
        conditions = []
        tables = []

        if dirList:
            dirString = intListToString(dirList)
            conditions.append("F.DirID in (%s)" % dirString)

        counter = 0
        for table, condition in tablesAndConditions:
            if table == "FC_FileInfo":
                query += "INNER JOIN FC_FileInfo FI USING( FileID ) "
                condition = condition.replace("%%", "%")
            elif table == "FC_Files":
                condition = condition.replace("%%", "%")
            else:
                counter += 1
                if table in leftJoinTables:
                    tables.append("LEFT JOIN %s M%d USING( FileID )" %
                                  (table, counter))
                else:
                    tables.append("INNER JOIN %s M%d USING( FileID )" %
                                  (table, counter))
                table = "M%d" % counter
                condition = condition % table
            conditions.append(condition)

        query += " ".join(tables)
        if conditions:
            query += " WHERE %s" % " AND ".join(conditions)

        result = self.db._query(query)
        if not result["OK"]:
            return result
        if not result["Value"]:
            return S_OK([])

        #     fileList = [ row[0] for row in result['Value' ] ]
        fileList = []
        for row in result["Value"]:
            fileID = row[0]
            fileList.append(fileID)

        return S_OK(fileList)
Exemplo n.º 39
0
 def _getRepIDsForReplica(self, replicaTuples, connection=False):
     connection = self._getConnection(connection)
     queryTuples = []
     for fileID, seID in replicaTuples:
         queryTuples.append("(%d,%d)" % (fileID, seID))
     req = "SELECT RepID,FileID,SEID FROM FC_Replicas WHERE (FileID,SEID) IN (%s)" % intListToString(
         queryTuples)
     res = self.db._query(req, connection)
     if not res['OK']:
         return res
     replicaDict = {}
     for repID, fileID, seID in res['Value']:
         replicaDict.setdefault(fileID, {})
         replicaDict[fileID][seID] = repID
     return S_OK(replicaDict)
Exemplo n.º 40
0
    def _getDirectoryFiles(self,
                           dirID,
                           fileNames,
                           metadata_input,
                           allStatus=False,
                           connection=False):
        """ Get the metadata for files in the same directory
    """
        metadata = list(metadata_input)

        connection = self._getConnection(connection)
        # metadata can be any of ['FileID','Size','UID','GID','Status','Checksum','CheckSumType',
        # 'Type','CreationDate','ModificationDate','Mode']
        req = "SELECT FileName,DirID,FileID,Size,UID,GID,Status FROM FC_Files WHERE DirID=%d" % (
            dirID)
        if not allStatus:
            statusIDs = []
            for status in self.db.visibleFileStatus:
                res = self._getStatusInt(status, connection=connection)
                if res['OK']:
                    statusIDs.append(res['Value'])
            if statusIDs:
                req = "%s AND Status IN (%s)" % (req,
                                                 intListToString(statusIDs))
        if fileNames:
            req = "%s AND FileName IN (%s)" % (req,
                                               stringListToString(fileNames))
        res = self.db._query(req, connection)
        if not res['OK']:
            return res
        fileNameIDs = res['Value']
        if not fileNameIDs:
            return S_OK({})
        filesDict = {}
        # If we only requested the FileIDs then there is no need to do anything else
        if metadata == ['FileID']:
            for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
                filesDict[fileName] = {'FileID': fileID}
            return S_OK(filesDict)
        # Otherwise get the additionally requested metadata from the FC_FileInfo table
        files = {}
        userDict = {}
        groupDict = {}
        for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
            filesDict[fileID] = fileName
            files[fileName] = {}
            if 'Size' in metadata:
                files[fileName]['Size'] = size
            if 'DirID' in metadata:
                files[fileName]['DirID'] = dirID
            if 'UID' in metadata:
                files[fileName]['UID'] = uid
                if uid in userDict:
                    owner = userDict[uid]
                else:
                    owner = 'unknown'
                    result = self.db.ugManager.getUserName(uid)
                    if result['OK']:
                        owner = result['Value']
                    userDict[uid] = owner
                files[fileName]['Owner'] = owner
            if 'GID' in metadata:
                files[fileName]['GID'] = gid
                if gid in groupDict:
                    group = groupDict[gid]
                else:
                    group = 'unknown'
                    result = self.db.ugManager.getGroupName(gid)
                    if result['OK']:
                        group = result['Value']
                    groupDict[gid] = group
                files[fileName]['OwnerGroup'] = group
            if 'Status' in metadata:
                files[fileName]['Status'] = status
        for element in ['FileID', 'Size', 'DirID', 'UID', 'GID', 'Status']:
            if element in metadata:
                metadata.remove(element)
        metadata.append('FileID')
        metadata.reverse()
        req = "SELECT %s FROM FC_FileInfo WHERE FileID IN (%s)" % (
            intListToString(metadata), intListToString(filesDict.keys()))
        res = self.db._query(req, connection)
        if not res['OK']:
            return res
        for tuple_ in res['Value']:
            fileID = tuple_[0]
            rowDict = dict(zip(metadata, tuple_))
            files[filesDict[fileID]].update(rowDict)
        return S_OK(files)
Exemplo n.º 41
0
 def setMigratingReplicaStatus(self,fileIDs,status):
   """ Update the status of the replica in the database
   """
   gLogger.info("setMigratingReplicaStatus: Attempting to update status of %d replicas to '%s'." % (len(fileIDs),status))
   req = "UPDATE MigratingReplicas SET Status='%s', LastUpdate=UTC_TIMESTAMP() WHERE ReplicaID IN (%s);" % (status,intListToString(fileIDs))
   res = self._update(req)
   if not res['OK']:
     gLogger.error("setMigratingReplicaStatus: Failed update replica statuses.",res['Message'])
   else:
     gLogger.info("setMigratingReplicaStatus: Successfully updated replica statuses.")
   return res
Exemplo n.º 42
0
 def _setFileParameter(self,fileID,paramName,paramValue,connection=False):
   connection = self._getConnection(connection)
   if type(fileID) not in [TupleType,ListType]:
     fileID = [fileID]
   req = "UPDATE FC_Files SET %s='%s', ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (%s)" % (paramName,paramValue,intListToString(fileID))
   return self.db._update(req,connection)
Exemplo n.º 43
0
 def getRequestFileStatus( self, requestID, files ):
   req = "SELECT DISTINCT SubRequestID FROM SubRequests WHERE RequestID = %d;" % requestID
   res = self._query( req )
   if not res['OK']:
     return res
   subRequests = []
   for subRequestID in res['Value'][0]:
     subRequests.append( subRequestID )
   req = "SELECT LFN,Status from Files WHERE SubRequestID IN (%s) AND LFN in (%s);" % ( intListToString( subRequests ), stringListToString( files ) )
   res = self._query( req )
   if not res['OK']:
     return res
   files = {}
   for lfn, status in res['Value']:
     files[lfn] = status
   return S_OK( files )