Пример #1
0
  def getDirectoryReplicas( self, dirID, path, allStatus = False, connection = False ):
    """
        This is defined in the FileManagerBase but it relies on the SEManager to get the SE names.
        It is good practice in software, but since the SE and Replica tables are bound together in the DB,
        I might as well resolve the name in the query


        Get the replicas for all the Files in the given Directory
        :param DirID : ID of the directory
        :param path : useless
        :param allStatus : whether all replicas and file status are considered
                          If False, take the visibleFileStatus and visibleReplicaStatus values from the configuration
    """

    # We format the visible file/replica satus so we can give it as argument to the ps
    # It is used in an IN clause, so it looks like --'"AprioriGood","Trash"'--
#     fStatus = ','.join( [ '"%s"' % status for status in self.db.visibleFileStatus ] )
#     rStatus = ','.join( [ '"%s"' % status for status in self.db.visibleReplicaStatus ] )
    fStatus = stringListToString( self.db.visibleFileStatus )
    rStatus = stringListToString( self.db.visibleReplicaStatus )

    result = self.db.executeStoredProcedureWithCursor( 'ps_get_replicas_for_files_in_dir', ( dirID, allStatus, fStatus, rStatus ) )
    if not result['OK']:
      return result


    resultDict = {}
    for fileName, _fileID, seName, pfn in result['Value']:
      resultDict.setdefault( fileName, {} ).setdefault( seName, [] ).append( pfn )

    return S_OK( resultDict )
Пример #2
0
  def _getDirectoryFiles(self,dirID,fileNames,metadata_input,allStatus=False,connection=False):
    """ For a given directory, and eventually given file, returns all the desired metadata

        :param dirID : directory ID
        :param filenames : the list of filenames, or []
        :param metadata_input: list of desired metadata.
                   It can be anything from (FileName, DirID, FileID, Size, UID, Owner,
                   GID, OwnerGroup, Status, GUID, Checksum, ChecksumType, Type, CreationDate, ModificationDate, Mode)
        :param allStatus : if False, only displays the files whose status is in db.visibleFileStatus

        :returns: S_OK(files), where files is a dictionary indexed on filename, and values are dictionary of metadata
    """

    connection = self._getConnection( connection )

    metadata = list( metadata_input )
    if "UID" in metadata:
      metadata.append( "Owner" )
    if "GID" in metadata:
      metadata.append( "OwnerGroup" )
    if "FileID" not in metadata:
      metadata.append( "FileID" )

    # Format the filenames and status to be used in a IN clause in the sotred procedure
    formatedFileNames = stringListToString( fileNames )
    fStatus = stringListToString( self.db.visibleFileStatus )

    specificFiles = True if len( fileNames ) else False
    result = self.db.executeStoredProcedureWithCursor( 'ps_get_all_info_for_files_in_dir',
                                                            ( dirID, specificFiles,
                                                             formatedFileNames, allStatus, fStatus ) )

    if not result['OK']:
      return result

    fieldNames = ["FileName", "DirID", "FileID", "Size", "UID", "Owner",
                  "GID", "OwnerGroup", "Status", "GUID", "Checksum",
                  "ChecksumType", "Type", "CreationDate", "ModificationDate", "Mode"]

    rows = result['Value']
    files = {}

    for row in rows:

      rowDict = dict( zip( fieldNames, row ) )
      fileName = rowDict['FileName']
      # Returns only the required metadata
      files[fileName] = dict( ( key, rowDict.get( key, "Unknown metadata field" ) ) for key in metadata )


    return S_OK( files )
Пример #3
0
  def getFTSFileList( self, statusList = None, limit = 1000 ):
    """ get at most :limit: FTSFiles with status in :statusList:

    :param statusList: list with FTSFiles statuses
    :type statusList: python:list
    :param int limit: select query limit
    """
    statusList = statusList if statusList else [ "Waiting" ]
    reStatus = []
    inStatus = []
    for status in statusList:
      if "%" in status or ".*" in status or ".+" in status:
        reStatus.append( status )
      else:
        inStatus.append( status )
    reQuery = "`Status` REGEXP '%s'" % "|".join( reStatus ) if reStatus else ""
    inQuery = "`Status` IN (%s)" % stringListToString( inStatus ) if inStatus else ""
    whereClause = " OR ".join( [ q for q in ( reQuery, inQuery ) if q ] )
    if whereClause:
      whereClause = "WHERE %s" % whereClause
    query = "SELECT * FROM `FTSFile` %s ORDER BY `LastUpdate` DESC LIMIT %s;" % ( whereClause, limit )
    trn = self._transaction( [query] )
    if not trn['OK']:
      self.log.error( "Failed getFTSFileList", "%s" % trn['Message'] )
      return trn
    return S_OK( [ FTSFile( fileDict ) for fileDict in trn['Value'][query] ] )
Пример #4
0
  def getDatasets( self, datasetName, credDict ):
    """ Get information about existing datasets
    """
    dsName = os.path.basename(datasetName)
    
    parameterList = ['DatasetID','MetaQuery','DirID','TotalSize','NumberOfFiles',
                     'UID','GID','Status','CreationDate','ModificationDate',
                     'DatasetHash','Mode','DatasetName']
    parameterString = ','.join( parameterList )

    req = "SELECT %s FROM FC_MetaDatasets" % parameterString
    if type( datasetName ) in StringTypes:
      dsName = os.path.basename(datasetName)
      if '*' in dsName:
        dName = dsName.replace( '*', '%' )
        req += " WHERE DatasetName LIKE '%s'" % dName
      elif dsName:
        req += " WHERE DatasetName='%s'" % dsName
    elif type( datasetName ) == ListType:
      dsNames = [ os.path.basename(d) for d in datasetName ]
      datasetString = stringListToString( dsNames )
      req += " WHERE DatasetName in (%s)" % datasetString

    result = self.db._query( req )
    if not result['OK']:
      return result

    resultDict = {}
    for row in result['Value']:
      dName = row[12]
      resultDict[dName] = self.__getDatasetDict( row )

    return S_OK( resultDict )
Пример #5
0
  def _getFileIDFromGUID( self, guids, connection = False ):
    """ Returns the file ids from list of guids
      :param guids : list of guid

      :returns dictionary  < guid : fileId >

    """

    connection = self._getConnection(connection)
    if not guids:
      return S_OK({})

    if type( guids ) not in [ListType, TupleType]:
      guids = [guids]

#     formatedGuids = ','.join( [ '"%s"' % guid for guid in guids ] )
    formatedGuids = stringListToString( guids )
    result = self.db.executeStoredProcedureWithCursor( 'ps_get_file_ids_from_guids', ( formatedGuids, ) )

    if not result['OK']:
      return result

    guidDict = dict( ( guid, fileID ) for guid, fileID in result['Value'] )

    return S_OK(guidDict)
Пример #6
0
 def _checkTaskUpdate( self, taskIDs, newTaskState, connection = False ):
   connection = self.__getConnection( connection )
   if not taskIDs:
     return S_OK( taskIDs )
   # * -> Failed
   if newTaskState == 'Failed':
     oldTaskState = []
   # StageCompleting -> Done
   elif newTaskState == 'Done':
     oldTaskState = ['StageCompleting']
   # StageSubmitted -> StageCompleting
   elif newTaskState == 'StageCompleting':
     oldTaskState = ['StageSubmitted']
   # Waiting -> StageSubmitted
   elif newTaskState == 'StageSubmitted':
     oldTaskState = ['Waiting', 'Offline']
   # New -> Waiting
   elif newTaskState == 'Waiting':
     oldTaskState = ['New']
   elif newTaskState == 'Offline':
     oldTaskState = ['Waiting']
   else:
     return S_ERROR( "Task status not recognized" )
   if not oldTaskState:
     toUpdate = taskIDs
   else:
     req = "SELECT TaskID FROM Tasks WHERE Status in (%s) AND TaskID IN (%s)" % ( stringListToString( oldTaskState ), intListToString( taskIDs ) )
     res = self._query( req, connection )
     if not res['OK']:
       return res
     toUpdate = [row[0] for row in res['Value']]
   return S_OK( toUpdate )
Пример #7
0
  def __findNoPathDatasets( self, nodirDatasets, connection ):

    failed = {}
    successful = {}
    dsIDs = {}
    req = "SELECT COUNT(DatasetName),DatasetName,DatasetID FROM FC_MetaDatasets WHERE DatasetName in "
    req += "( %s ) GROUP BY DatasetName" % stringListToString( nodirDatasets )
    result = self.db._query( req, connection )
    if not result['OK']:
      return result
    for dsCount, dsName, dsID in result['Value']:
      if dsCount > 1:
        failed[dsName] = "Ambiguous dataset name"
      else:  
        dsIDs[dsName] = str( dsID )
        
    if dsIDs:
      req = "SELECT DatasetName,DatasetID,DirID FROM FC_MetaDatasets WHERE DatasetID in (%s)" % ','.join( dsIDs.values() ) 
      result = self.db._query( req, connection )
      if not result['OK']:
        return result 
      for dsName, dsID, dirID in result['Value']:
        successful[dsName] = { "DirID": dirID, "DatasetID": dsID }   
       
    for name in nodirDatasets:
      if name not in failed and name not in successful:
        failed[name] = "Dataset not found"     
        
    return S_OK({"Successful":successful,"Failed":failed})    
Пример #8
0
  def getFTSJobsForRequest( self, requestID, statusList = None ):
    """ get list of FTSJobs with status in :statusList: for request given its requestID

    TODO: should be more smart, i.e. one query to select all ftsfiles
    """

    statusList = statusList if statusList  else list( FTSJob.INITSTATES + FTSJob.TRANSSTATES )
    query = "SELECT * FROM `FTSJob` WHERE `RequestID` = %s AND `Status` in (%s)" % ( requestID,
                                                                                     stringListToString( statusList ) )
    ftsJobs = self._transaction( [ query ] )
    if not ftsJobs['OK']:
      self.log.error( "Failed getFTSJobsForRequest", "%s" % ftsJobs['Message'] )
      return ftsJobs

    ftsJobs = ftsJobs['Value'][query] if query in ftsJobs['Value'] else []

    ftsJobs = [ FTSJob( ftsJobDict ) for ftsJobDict in ftsJobs  ]

    for ftsJob in ftsJobs:
      query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s' AND `RequestID`=%s" % ( ftsJob.FTSGUID,
                                                                                      requestID )
      ftsFiles = self._transaction( [ query ] )
      if not ftsFiles['OK']:
        self.log.error( "Failed getFTSJobsForRequest", "%s" % ftsFiles['Message'] )
        return ftsFiles
      ftsFiles = ftsFiles['Value'][query] if query in ftsFiles['Value'] else []
      for ftsFileDict in ftsFiles:
        ftsJob.addFile( FTSFile( ftsFileDict ) )
    return S_OK( ftsJobs )
Пример #9
0
 def getFTSFileIDs( self, statusList = None ):
   """ select FTSFileIDs for a given status list """
   statusList = statusList if statusList else [ "Waiting" ]
   query = "SELECT `FTSFileID` FROM `FTSFile` WHERE `Status` IN (%s);" % stringListToString( statusList );
   query = self._query( query )
   if not query['OK']:
     self.log.error( 'Failed ftsFileSQL', query['Message'] )
     return query
   return S_OK( [ item[0] for item in query['Value'] ] )
Пример #10
0
 def getFTSJobIDs( self, statusList = [ "Submitted", "Active", "Ready" ] ):
   """ get FTSJobIDs for  a given status list """
   query = "SELECT `FTSJobID` FROM `FTSJob` WHERE `Status` IN (%s);" % stringListToString( statusList )
   query = self._query( query )
   if not query['OK']:
     self.log.error( 'Failed ftsJobSQL', query['Message'] )
     return query
   # # convert to list of longs
   return S_OK( [ item[0] for item in query['Value'] ] )
Пример #11
0
 def getFTSJobIDs(self, statusList=["Submitted", "Active", "Ready"]):
     """ get FTSJobIDs for  a given status list """
     query = "SELECT `FTSJobID` FROM `FTSJob` WHERE `Status` IN (%s);" % stringListToString(
         statusList)
     query = self._query(query)
     if not query['OK']:
         self.log.error('Failed ftsJobSQL', query['Message'])
         return query
     # # convert to list of longs
     return S_OK([item[0] for item in query['Value']])
Пример #12
0
 def getFTSFileIDs(self, statusList=None):
     """ select FTSFileIDs for a given status list """
     statusList = statusList if statusList else ["Waiting"]
     query = "SELECT `FTSFileID` FROM `FTSFile` WHERE `Status` IN (%s);" % stringListToString(
         statusList)
     query = self._query(query)
     if not query['OK']:
         self.log.error('Failed ftsFileSQL', query['Message'])
         return query
     return S_OK([item[0] for item in query['Value']])
Пример #13
0
    def _findFileIDs(self, lfns, connection=False):
        """Find lfn <-> FileID correspondence"""
        connection = self._getConnection(connection)
        failed = {}
        successful = {}

        # If there is only one lfn, we might as well make a direct query
        if len(lfns) == 1:
            lfn = list(lfns)[0]  # if lfns is a dict, list(lfns) returns lfns.keys()
            pathPart, filePart = os.path.split(lfn)
            result = self.db.executeStoredProcedure(
                "ps_get_file_id_from_lfn", (pathPart, filePart, "ret1"), outputIds=[2]
            )
            if not result["OK"]:
                return result

            fileId = result["Value"][0]

            if not fileId:
                failed[lfn] = "No such file or directory"
            else:
                successful[lfn] = fileId

        else:

            # We separate the files by directory
            filesInDirDict = self._getFileDirectories(lfns)

            # We get the directory ids
            result = self.db.dtree.findDirs(list(filesInDirDict))
            if not result["OK"]:
                return result
            directoryPathToIds = result["Value"]

            # For each directory, we get the file ids of the files we want
            for dirPath in directoryPathToIds:
                fileNames = filesInDirDict[dirPath]
                dirID = directoryPathToIds[dirPath]

                formatedFileNames = stringListToString(fileNames)

                result = self.db.executeStoredProcedureWithCursor(
                    "ps_get_file_ids_from_dir_id", (dirID, formatedFileNames)
                )
                if not result["OK"]:
                    return result
                for fileID, fileName in result["Value"]:
                    fname = os.path.join(dirPath, fileName)
                    successful[fname] = fileID

            # The lfns that are not in successful dont exist
            for failedLfn in set(lfns) - set(successful):
                failed[failedLfn] = "No such file or directory"

        return S_OK({"Successful": successful, "Failed": failed})
Пример #14
0
    def _getFileReplicas(self,
                         fileIDs,
                         fields_input=['PFN'],
                         allStatus=False,
                         connection=False):
        """ Get replicas for the given list of files specified by their fileIDs
        :param fileIDs : list of file ids
        :param fields_input : metadata of the Replicas we are interested in
        :param allStatus : if True, all the Replica statuses will be considered, otherwise, only the db.visibleReplicaStatus

        :returns S_OK with a dict { fileID : { SE name : dict of metadata } }
    """

        connection = self._getConnection(connection)

        fields = list(fields_input)

        if 'Status' not in fields:
            fields.append('Status')

        replicas = {}

        # Format the status to be used in a IN clause in the stored procedure
        fStatus = stringListToString(self.db.visibleReplicaStatus)

        fieldNames = [
            "FileID", "SE", "Status", "RepType", "CreationDate",
            "ModificationDate", "PFN"
        ]

        for fileID in fileIDs:

            result = self.db.executeStoredProcedureWithCursor(
                'ps_get_all_info_of_replicas', (fileID, allStatus, fStatus))

            if not result['OK']:
                return result

            rows = result['Value']

            if not rows:
                replicas[fileID] = {}

            for row in rows:

                rowDict = dict(zip(fieldNames, row))

                # Returns only the required metadata
                se = rowDict["SE"]
                repForFile = replicas.setdefault(fileID, {})
                repForFile[se] = dict(
                    (key, rowDict.get(key, "Unknown metadata field"))
                    for key in fields)

        return S_OK(replicas)
Пример #15
0
 def removeMigratingFiles(self,lfns):
   """ Remove all replicas with associated LFNs from the database
   """
   gLogger.info("removeMigratingFiles: Attempting to remove %d files from the database." % len(lfns))
   req = "DELETE FROM MigratingReplicas WHERE LFN IN (%s);" % stringListToString(lfns)
   res = self._update(req)
   if not res['OK']:
     gLogger.error("removeMigratingFiles: Failed removing files from database.",res['Message'])
   else:
     gLogger.info("removeMigratingFiles: Successfully removed files.")
   return res
Пример #16
0
  def _findFileIDs(self, lfns, connection=False):
    """ Find lfn <-> FileID correspondence
    """
    connection = self._getConnection(connection)
    failed = {}
    successful = {}

    # If there is only one lfn, we might as well make a direct query
    if len(lfns) == 1:
      lfn = list(lfns)[0]  # if lfns is a dict, list(lfns) returns lfns.keys()
      pathPart, filePart = os.path.split(lfn)
      result = self.db.executeStoredProcedure(
          'ps_get_file_id_from_lfn', (pathPart, filePart, 'ret1'), outputIds=[2])
      if not result['OK']:
        return result

      fileId = result['Value'][0]

      if not fileId:
        failed[lfn] = "No such file or directory"
      else:
        successful[lfn] = fileId

    else:

      # We separate the files by directory
      filesInDirDict = self._getFileDirectories(lfns)

      # We get the directory ids
      result = self.db.dtree.findDirs(filesInDirDict.keys())
      if not result['OK']:
        return result
      directoryPathToIds = result['Value']

      # For each directory, we get the file ids of the files we want
      for dirPath in directoryPathToIds:
        fileNames = filesInDirDict[dirPath]
        dirID = directoryPathToIds[dirPath]

        formatedFileNames = stringListToString(fileNames)

        result = self.db.executeStoredProcedureWithCursor(
            'ps_get_file_ids_from_dir_id', (dirID, formatedFileNames))
        if not result['OK']:
          return result
        for fileID, fileName in result['Value']:
          fname = os.path.join(dirPath, fileName)
          successful[fname] = fileID

      # The lfns that are not in successful dont exist
      for failedLfn in (set(lfns) - set(successful)):
        failed[failedLfn] = "No such file or directory"

    return S_OK({"Successful": successful, "Failed": failed})
Пример #17
0
 def getRequestNamesList( self, statusList = None, limit = None ):
   """ select requests with status in :statusList: """
   statusList = statusList if statusList else list( Request.FINAL_STATES )
   limit = limit if limit else 100
   query = "SELECT `RequestName`, `Status`, `LastUpdate` FROM `Request` WHERE "\
     " `Status` IN (%s) ORDER BY `LastUpdate` DESC LIMIT %s;" % ( stringListToString( statusList ), limit )
   reqNamesList = self._query( query )
   if not reqNamesList["OK"]:
     self.log.error( "getRequestNamesList: %s" % reqNamesList["Message"] )
     return reqNamesList
   reqNamesList = reqNamesList["Value"]
   return S_OK( [ reqName for reqName in reqNamesList] )
Пример #18
0
 def getFTSFilesForRequest(self, requestID, statusList=None):
     """ get FTSFiles with status in :statusList: for request given its :requestID: """
     requestID = int(requestID)
     statusList = statusList if statusList else ["Waiting"]
     query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s AND `Status` IN (%s);" % (
         requestID, stringListToString(statusList))
     ftsFiles = self._transaction([query])
     if not ftsFiles['OK']:
         self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message'])
         return ftsFiles
     ftsFiles = ftsFiles['Value'].get(query, [])
     return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
Пример #19
0
    def getDirectoryReplicas(self,
                             dirID,
                             path,
                             allStatus=False,
                             connection=False):
        """
        This is defined in the FileManagerBase but it relies on the SEManager to get the SE names.
        It is good practice in software, but since the SE and Replica tables are bound together in the DB,
        I might as well resolve the name in the query


        Get the replicas for all the Files in the given Directory

        :param int dirID: ID of the directory
        :param unused path: useless
        :param bool allStatus: whether all replicas and file status are considered
                               If False, take the visibleFileStatus and visibleReplicaStatus
                               values from the configuration
        """

        # We format the visible file/replica satus so we can give it as argument to the ps
        # It is used in an IN clause, so it looks like --'"AprioriGood","Trash"'--
        #     fStatus = ','.join( [ '"%s"' % status for status in self.db.visibleFileStatus ] )
        #     rStatus = ','.join( [ '"%s"' % status for status in self.db.visibleReplicaStatus ] )
        fStatus = stringListToString(self.db.visibleFileStatus)
        rStatus = stringListToString(self.db.visibleReplicaStatus)

        result = self.db.executeStoredProcedureWithCursor(
            "ps_get_replicas_for_files_in_dir",
            (dirID, allStatus, fStatus, rStatus))
        if not result["OK"]:
            return result

        resultDict = {}
        for fileName, _fileID, seName, pfn in result["Value"]:
            resultDict.setdefault(fileName, {}).setdefault(seName,
                                                           []).append(pfn)

        return S_OK(resultDict)
Пример #20
0
 def __findDirs(self, paths, metadata=['DirName']):
     dirs = {}
     req = "SELECT DirID,%s FROM DirectoryInfo WHERE DirName IN (%s)" % (
         intListToString(metadata), stringListToString(paths))
     res = self.db._query(req)
     if not res['OK']:
         return res
     if not res['Value']:
         return S_OK(dirs)
     for tup in res['Value']:
         dirID = tup[0]
         dirs[dirID] = dict(zip(metadata, tup[1:]))
     return S_OK(dirs)
Пример #21
0
 def _getFileIDFromGUID(self, guid, connection=False):
   connection = self._getConnection(connection)
   if not guid:
     return S_OK({})
   if not isinstance(guid, (list, tuple)):
     guid = [guid]
   req = "SELECT FileID,GUID FROM FC_Files WHERE GUID IN (%s)" % stringListToString(guid)
   res = self.db._query(req, connection)
   if not res['OK']:
     return res
   guidDict = {}
   for fileID, guid in res['Value']:
     guidDict[guid] = fileID
   return S_OK(guidDict)
Пример #22
0
 def getRequestNamesList( self, statusList = None, limit = None, since = None, until = None ):
   """ select requests with status in :statusList: """
   statusList = statusList if statusList else list( Request.FINAL_STATES )
   limit = limit if limit else 100
   sinceReq = " AND LastUpdate > %s " % since  if since else ""
   untilReq = " AND LastUpdate < %s " % until if until else ""
   query = "SELECT `RequestName`, `Status`, `LastUpdate` FROM `Request` WHERE "\
     " `Status` IN (%s) %s %s ORDER BY `LastUpdate` ASC LIMIT %s;" % ( stringListToString( statusList ), sinceReq, untilReq, limit )
   reqNamesList = self._query( query )
   if not reqNamesList["OK"]:
     self.log.error( "getRequestNamesList: %s" % reqNamesList["Message"] )
     return reqNamesList
   reqNamesList = reqNamesList["Value"]
   return S_OK( [ reqName for reqName in reqNamesList] )
Пример #23
0
 def _getFileIDFromGUID(self,guid,connection=False):
   connection = self._getConnection(connection)
   if not guid:
     return S_OK({})
   if type(guid) not in [ListType,TupleType]:
     guid = [guid] 
   req = "SELECT FileID,GUID FROM FC_FileInfo WHERE GUID IN (%s)" % stringListToString(guid)
   res = self.db._query(req,connection)
   if not res['OK']:
     return res
   guidDict = {}
   for fileID,guid in res['Value']:
     guidDict[guid] = fileID
   return S_OK(guidDict)
Пример #24
0
  def _getFileReplicas( self, fileIDs, fields_input = ['PFN'], allStatus = False, connection = False ):
    """ Get replicas for the given list of files specified by their fileIDs
        :param fileIDs : list of file ids
        :param fields_input : metadata of the Replicas we are interested in
        :param allStatus : if True, all the Replica statuses will be considered, otherwise, only the db.visibleReplicaStatus

        :returns S_OK with a dict { fileID : { SE name : dict of metadata } }
    """

    connection = self._getConnection( connection )


    fields = list( fields_input )

    if 'Status' not in fields:
      fields.append( 'Status' )

    replicas = {}

    # Format the status to be used in a IN clause in the stored procedure
    fStatus = stringListToString( self.db.visibleReplicaStatus )


    fieldNames = [ "FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"]

    for fileID in fileIDs:

      result = self.db.executeStoredProcedureWithCursor( 'ps_get_all_info_of_replicas',
                                                            ( fileID, allStatus, fStatus ) )

      if not result['OK']:
        return result


      rows = result['Value']

      if not rows:
        replicas[fileID] = {}

      for row in rows:

        rowDict = dict( zip( fieldNames, row ) )

        # Returns only the required metadata
        se = rowDict["SE"]
        repForFile = replicas.setdefault( fileID, {} )
        repForFile[se] = dict( ( key, rowDict.get( key, "Unknown metadata field" ) ) for key in fields )

    return S_OK(replicas)
Пример #25
0
    def _getFileReplicas(self, fileIDs, fields_input=None, allStatus=False, connection=False):
        """Get replicas for the given list of files specified by their fileIDs
        :param fileIDs : list of file ids
        :param fields_input : metadata of the Replicas we are interested in (default to PFN)
        :param allStatus : if True, all the Replica statuses will be considered,
                           otherwise, only the db.visibleReplicaStatus

        :returns S_OK with a dict { fileID : { SE name : dict of metadata } }
        """

        if fields_input is None:
            fields_input = ["PFN"]

        fields = list(fields_input)

        # always add Status in the list of required fields
        if "Status" not in fields:
            fields.append("Status")

        # We initialize the dictionary with empty dict
        # as default value, because this is what we want for
        # non existing replicas
        replicas = {fileID: {} for fileID in fileIDs}

        # Format the status to be used in a IN clause in the stored procedure
        fStatus = stringListToString(self.db.visibleReplicaStatus)

        fieldNames = ["FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"]

        for chunks in breakListIntoChunks(fileIDs, 1000):
            # Format the FileIDs to be used in a IN clause in the stored procedure
            formatedFileIds = intListToString(chunks)
            result = self.db.executeStoredProcedureWithCursor(
                "ps_get_all_info_of_replicas_bulk", (formatedFileIds, allStatus, fStatus)
            )

            if not result["OK"]:
                return result

            rows = result["Value"]

            for row in rows:
                rowDict = dict(zip(fieldNames, row))
                se = rowDict["SE"]
                fileID = rowDict["FileID"]
                replicas[fileID][se] = dict((key, rowDict.get(key, "Unknown metadata field")) for key in fields)

        return S_OK(replicas)
Пример #26
0
 def __deleteFiles(self, fileIDs, connection=False):
     connection = self._getConnection(connection)
     if type(fileIDs) not in [ListType, TupleType]:
         fileIDs = [fileIDs]
     if not fileIDs:
         return S_OK()
     fileIDString = intListToString(fileIDs)
     failed = []
     for table in ["FC_Files", "FC_FileInfo"]:
         req = "DELETE FROM %s WHERE FileID in (%s)" % (table, fileIDString)
         res = self.db._update(req, connection)
         if not res["OK"]:
             gLogger.error("Failed to remove files from table %s" % table, res["Message"])
             failed.append(table)
     if failed:
         return S_ERROR("Failed to remove files from %s" % stringListToString(failed))
     return S_OK()
Пример #27
0
 def getRequestFileStatus(self, requestID, files):
     req = "SELECT DISTINCT SubRequestID FROM SubRequests WHERE RequestID = %d;" % requestID
     res = self._query(req)
     if not res['OK']:
         return res
     subRequests = []
     for subRequestID in res['Value'][0]:
         subRequests.append(subRequestID)
     req = "SELECT LFN,Status from Files WHERE SubRequestID IN (%s) AND LFN in (%s);" % (
         intListToString(subRequests), stringListToString(files))
     res = self._query(req)
     if not res['OK']:
         return res
     files = {}
     for lfn, status in res['Value']:
         files[lfn] = status
     return S_OK(files)
Пример #28
0
 def __deleteFiles( self, fileIDs, connection = False ):
   connection = self._getConnection(connection)
   if type(fileIDs) not in [ ListType, TupleType]:
     fileIDs = [fileIDs]
   if not fileIDs:
     return S_OK()
   fileIDString = intListToString( fileIDs )
   failed = []
   for table in ['FC_Files','FC_FileInfo']:
     req = "DELETE FROM %s WHERE FileID in (%s)" % ( table, fileIDString )
     res = self.db._update(req,connection)
     if not res['OK']:
       gLogger.error( "Failed to remove files from table %s" % table, res['Message'] )
       failed.append(table)
   if failed:
     return S_ERROR( "Failed to remove files from %s" % stringListToString( failed ) )
   return S_OK()
Пример #29
0
 def __deleteReplicas(self,repIDs,connection=False):
   connection = self._getConnection(connection)
   if type(repIDs) not in [ ListType, TupleType]:
     repIDs = [repIDs]
   if not repIDs:
     return S_OK()
   repIDString = intListToString(repIDs)
   failed = []
   for table in ['FC_Replicas','FC_ReplicaInfo']:
     req = "DELETE FROM %s WHERE RepID in (%s)" % (table,repIDString)
     res = self.db._update(req,connection)
     if not res['OK']:
       gLogger.error("Failed to remove replicas from %s" % table,res['Message'])
       failed.append(table)
   if failed:
     return S_ERROR("Failed to remove replicas from %s" % stringListToString(failed))
   return S_OK()
Пример #30
0
 def __deleteReplicas(self, repIDs, connection=False):
   connection = self._getConnection(connection)
   if not isinstance(repIDs, (list, tuple)):
     repIDs = [repIDs]
   if not repIDs:
     return S_OK()
   repIDString = intListToString(repIDs)
   failed = []
   for table in ['FC_Replicas', 'FC_ReplicaInfo']:
     req = "DELETE FROM %s WHERE RepID in (%s)" % (table, repIDString)
     res = self.db._update(req, connection)
     if not res['OK']:
       gLogger.error("Failed to remove replicas from table %s" % table, res['Message'])
       failed.append(table)
   if failed:
     return S_ERROR("Failed to remove replicas from %s" % stringListToString(failed))
   return S_OK()
Пример #31
0
    def _findFileIDs(self, lfns, connection=False):
        """ Find lfn <-> FileID correspondence
    """
        connection = self._getConnection(connection)
        dirDict = self._getFileDirectories(lfns)
        failed = {}
        successful = {}
        result = self.db.dtree.findDirs(dirDict.keys())
        if not result['OK']:
            return result
        directoryIDs = result['Value']
        directoryPaths = {}

        for dirPath in dirDict:
            if not dirPath in directoryIDs:
                for fileName in dirDict[dirPath]:
                    fname = '%s/%s' % (dirPath, fileName)
                    fname = fname.replace('//', '/')
                    failed[fname] = 'No such directory'
            else:
                directoryPaths[directoryIDs[dirPath]] = dirPath
        directoryIDList = directoryIDs.keys()
        for dirIDs in breakListIntoChunks(directoryIDList, 1000):

            wheres = []
            for dirPath in dirIDs:
                fileNames = dirDict[dirPath]
                dirID = directoryIDs[dirPath]
                wheres.append("( DirID=%d AND FileName IN (%s) )" %
                              (dirID, stringListToString(fileNames)))

            req = "SELECT FileName,DirID,FileID FROM FC_Files WHERE %s" % " OR ".join(
                wheres)
            result = self.db._query(req, connection)
            if not result['OK']:
                return result
            for fileName, dirID, fileID in result['Value']:
                fname = '%s/%s' % (directoryPaths[dirID], fileName)
                fname = fname.replace('//', '/')
                successful[fname] = fileID

        for lfn in lfns:
            if not lfn in successful:
                failed[lfn] = "No such file"

        return S_OK({"Successful": successful, "Failed": failed})
Пример #32
0
  def findDirs( self, paths, connection = False ):
    """ Find DirIDs for the given path list
        :param paths list of path

        :returns S_OK( { path : ID} )
    """

    dirDict = {}
    if not paths:
      return S_OK( dirDict )
    dpaths = stringListToString( [os.path.normpath( path ) for path in paths ] )
    result = self.db.executeStoredProcedureWithCursor( 'ps_find_dirs', ( dpaths, ) )
    if not result['OK']:
      return result
    for dirName, dirID in result['Value']:
      dirDict[dirName] = dirID

    return S_OK( dirDict )
Пример #33
0
 def getRequestNamesList(self,
                         statusList=None,
                         limit=None,
                         since=None,
                         until=None):
     """ select requests with status in :statusList: """
     statusList = statusList if statusList else list(Request.FINAL_STATES)
     limit = limit if limit else 100
     sinceReq = " AND LastUpdate > %s " % since if since else ""
     untilReq = " AND LastUpdate < %s " % until if until else ""
     query = "SELECT `RequestName`, `Status`, `LastUpdate` FROM `Request` WHERE "\
       " `Status` IN (%s) %s %s ORDER BY `LastUpdate` ASC LIMIT %s;" % ( stringListToString( statusList ), sinceReq, untilReq, limit )
     reqNamesList = self._query(query)
     if not reqNamesList["OK"]:
         self.log.error("getRequestNamesList: %s" % reqNamesList["Message"])
         return reqNamesList
     reqNamesList = reqNamesList["Value"]
     return S_OK([reqName for reqName in reqNamesList])
Пример #34
0
  def _findFileIDs( self, lfns, connection=False ):
    """ Find lfn <-> FileID correspondence
    """
    connection = self._getConnection(connection)
    dirDict = self._getFileDirectories(lfns)
    failed = {}
    successful = {}
    result = self.db.dtree.findDirs( dirDict.keys() )
    if not result['OK']:
      return result
    directoryIDs = result['Value']
    directoryPaths = {}

    for dirPath in dirDict:
      if not dirPath in directoryIDs:
        for fileName in dirDict[dirPath]:
          fname = '%s/%s' % (dirPath,fileName)
          fname = fname.replace('//','/')
          failed[fname] = 'No such file or directory'
      else:
        directoryPaths[directoryIDs[dirPath]] = dirPath
    directoryIDList = directoryIDs.keys()
    for dirIDs in breakListIntoChunks( directoryIDList, 1000 ):

      wheres = []
      for dirPath in dirIDs:
        fileNames = dirDict[dirPath]
        dirID = directoryIDs[dirPath]
        wheres.append( "( DirID=%d AND FileName IN (%s) )" % (dirID, stringListToString(fileNames) ) )

      req = "SELECT FileName,DirID,FileID FROM FC_Files WHERE %s" % " OR ".join( wheres )
      result = self.db._query(req,connection)
      if not result['OK']:
        return result
      for fileName, dirID, fileID in result['Value']:
        fname = '%s/%s' % (directoryPaths[dirID],fileName)
        fname = fname.replace('//','/')
        successful[fname] = fileID

    for lfn in lfns:
      if not lfn in successful:
        failed[lfn] = "No such file or directory"

    return S_OK({"Successful":successful,"Failed":failed})
Пример #35
0
    def getFiles(self, status):
        """ Obtain all the files in a give status (or list) in the database along with all their associated metadata

        :param status: string or list of status we want to query

        :returns: dict {lfn: metadata}
    """
        if isinstance(status, basestring):
            status = [status]

        try:
            gLogger.info(
                "RAWIntegrityDB.getActiveFiles: Obtaining files in status %s."
                % status)
            req = "SELECT LFN,Status,PFN,Size,StorageElement,GUID,FileChecksum,TIME_TO_SEC(TIMEDIFF(UTC_TIMESTAMP(),SubmitTime)),SubmitTime from Files WHERE Status in (%s);" % (
                stringListToString(status))
            res = self._query(req)
            if not res['OK']:
                gLogger.error(
                    "RAWIntegrityDB.getActiveFiles: Failed to get files from database.",
                    res['Message'])
                return res
            fileDict = {}
            for lfn, st, pfn, size, se, guid, checksum, waittime, submitTime in res[
                    'Value']:
                fileDict[lfn] = {
                    'PFN': pfn,
                    'Size': size,
                    'SE': se,
                    'GUID': guid,
                    'Checksum': checksum,
                    'WaitTime': waittime,
                    'Status': st,
                    'SubmitTime': submitTime
                }
            gLogger.info(
                "RAWIntegrityDB.getActiveFiles: Obtained %s files awaiting migration from database."
                % len(fileDict.keys()))
            return S_OK(fileDict)
        except Exception as x:
            errStr = "RAWIntegrityDB.getActiveFiles: Exception while getting files from database."
            gLogger.exception(errStr, lException=x)
            return S_ERROR(errStr)
Пример #36
0
  def getLFNForGUID(self, guids, connection=False):
    """ Returns the lfns matching given guids"""
    connection = self._getConnection(connection)
    if not guids:
      return S_OK({})

    if not isinstance(guids, (list, tuple)):
      guids = [guids]

    formatedGuids = stringListToString(guids)
    result = self.db.executeStoredProcedureWithCursor('ps_get_lfns_from_guids', (formatedGuids, ))

    if not result['OK']:
      return result

    guidDict = dict((guid, lfn) for guid, lfn in result['Value'])
    failedGuid = set(guids) - set(guidDict)
    failed = dict.fromkeys(failedGuid, "GUID does not exist") if failedGuid else {}
    return S_OK({"Successful": guidDict, "Failed": failed})
Пример #37
0
    def findDirs(self, paths, connection=False):
        """ Find DirIDs for the given path list
        :param paths list of path

        :returns S_OK( { path : ID} )
    """

        dirDict = {}
        if not paths:
            return S_OK(dirDict)
        dpaths = stringListToString([os.path.normpath(path) for path in paths])
        result = self.db.executeStoredProcedureWithCursor(
            'ps_find_dirs', (dpaths, ))
        if not result['OK']:
            return result
        for dirName, dirID in result['Value']:
            dirDict[dirName] = dirID

        return S_OK(dirDict)
Пример #38
0
 def __deleteFiles(self, fileIDs, connection=False):
     connection = self._getConnection(connection)
     if not isinstance(fileIDs, (list, tuple)):
         fileIDs = [fileIDs]
     if not fileIDs:
         return S_OK()
     fileIDString = intListToString(fileIDs)
     failed = []
     for table in ["FC_Files", "FC_FileInfo"]:
         req = "DELETE FROM %s WHERE FileID in (%s)" % (table, fileIDString)
         res = self.db._update(req, connection)
         if not res["OK"]:
             gLogger.error("Failed to remove files from table %s" % table,
                           res["Message"])
             failed.append(table)
     if failed:
         return S_ERROR("Failed to remove files from %s" %
                        stringListToString(failed))
     return S_OK()
Пример #39
0
  def getLFNForGUID( self, guids, connection = False ):
    """ Returns the lfns matching given guids"""
    connection = self._getConnection( connection )
    if not guids:
      return S_OK( {} )

    if type( guids ) not in [ListType, TupleType]:
      guids = [guids]

    formatedGuids = stringListToString( guids )
    result = self.db.executeStoredProcedureWithCursor( 'ps_get_lfns_from_guids', ( formatedGuids, ) )

    if not result['OK']:
      return result

    guidDict = dict( ( guid, lfn ) for guid, lfn in result['Value'] )
    failedGuid = set( guids ) - set( guidDict )
    failed = dict.fromkeys( failedGuid, "GUID does not exist" ) if failedGuid else {}
    return S_OK( {"Successful" : guidDict, "Failed" : failed} )
Пример #40
0
    def __findFullPathDatasets(self, datasets, connection):

        dirDict = self._getDatasetDirectories(datasets)
        failed = {}
        successful = {}
        result = self.db.dtree.findDirs(list(dirDict))
        if not result["OK"]:
            return result
        directoryIDs = result["Value"]

        directoryPaths = {}
        for dirPath in dirDict:
            if dirPath not in directoryIDs:
                for dsName in dirDict[dirPath]:
                    dname = "%s/%s" % (dirPath, dsName)
                    dname = dname.replace("//", "/")
                    failed[dname] = "No such dataset or directory"
            else:
                directoryPaths[directoryIDs[dirPath]] = dirPath

        wheres = []
        for dirPath in directoryIDs:
            dsNames = dirDict[dirPath]
            dirID = directoryIDs[dirPath]
            wheres.append("( DirID=%d AND DatasetName IN (%s) )" %
                          (dirID, stringListToString(dsNames)))

        req = "SELECT DatasetName,DirID,DatasetID FROM FC_MetaDatasets WHERE %s" % " OR ".join(
            wheres)
        result = self.db._query(req, connection)
        if not result["OK"]:
            return result
        for dsName, dirID, dsID in result["Value"]:
            dname = "%s/%s" % (directoryPaths[dirID], dsName)
            dname = dname.replace("//", "/")
            successful[dname] = {"DirID": dirID, "DatasetID": dsID}

        for dataset in datasets:
            if dataset not in successful:
                failed[dataset] = "No such dataset"

        return S_OK({"Successful": successful, "Failed": failed})
Пример #41
0
    def getLFNForGUID(self, guids, connection=False):
        """ Returns the lfns matching given guids"""

        connection = self._getConnection(connection)
        if not guids:
            return S_OK({})
        if type(guids) not in [ListType, TupleType]:
            guids = [guids]
        req = (
            "SELECT f.FileID, f.FileName, fi.GUID, f.DirID FROM FC_FileInfo fi JOIN FC_Files f on fi.FileID = f.FileID WHERE GUID IN (%s)"
            % stringListToString(guids)
        )
        res = self.db._query(req, connection)
        if not res["OK"]:
            return res

        fileIDguid = {}
        dirIDFileIDs = {}
        fileIDName = {}
        for fileID, fileName, guid, dirID in res["Value"]:
            fileIDguid[fileID] = guid
            dirIDFileIDs.setdefault(dirID, []).append(fileID)
            fileIDName[fileID] = fileName

        res = self.db.dtree.getDirectoryPaths(dirIDFileIDs.keys())
        if not res["OK"]:
            return res

        dirIDName = res["Value"]

        guidLFN = dict(
            [
                (fileIDguid[fId], os.path.join(dirIDName[dId], fileIDName[fId]))
                for dId in dirIDName
                for fId in dirIDFileIDs[dId]
            ]
        )
        failedGuid = set(guids) - set(guidLFN)
        failed = dict.fromkeys(failedGuid, "GUID does not exist") if failedGuid else {}

        return S_OK({"Successful": guidLFN, "Failed": failed})
Пример #42
0
 def _findDirectories(self, paths, metadata=[]):
   """ Find file ID if it exists for the given list of LFNs """
   successful = {}
   failed = {}
   req = "SELECT DirName,DirID"
   if metadata:
     req = "%s,%s" % (req, intListToString(metadata))
   req = "%s FROM DirectoryInfo WHERE DirName IN (%s)" % (req, stringListToString(paths))
   res = self.db._query(req)
   if not res['OK']:
     return res
   for tup in res['Value']:
     dirName = tup[0]
     dirID = tup[1]
     metaDict = {'DirID': dirID}
     metaDict.update(dict(zip(metadata, tup[2:])))
     successful[dirName] = metaDict
   for path in paths:
     if path not in successful:
       failed[path] = 'No such file or directory'
   return S_OK({"Successful": successful, "Failed": failed})
Пример #43
0
 def _getDirectoryFiles(self, dirID, fileNames, metadata, allStatus=False, connection=False):
     connection = self._getConnection(connection)
     # metadata can be any of ['FileID','Size','UID','GID','Checksum','ChecksumType','Type','CreationDate','ModificationDate','Mode','Status']
     req = "SELECT FileName,%s FROM FC_Files WHERE DirID=%d" % (intListToString(metadata), dirID)
     if not allStatus:
         statusIDs = []
         res = self._getStatusInt("AprioriGood", connection=connection)
         if res["OK"]:
             statusIDs.append(res["Value"])
         if statusIDs:
             req = "%s AND Status IN (%s)" % (req, intListToString(statusIDs))
     if fileNames:
         req = "%s AND FileName IN (%s)" % (req, stringListToString(fileNames))
     res = self.db._query(req, connection)
     if not res["OK"]:
         return res
     files = {}
     for tuple in res["Value"]:
         fileName = tuple[0]
         files[fileName] = dict(zip(metadata, tuple[1:]))
     return S_OK(files)
Пример #44
0
 def _getDirectoryFiles(self,dirID,fileNames,metadata,allStatus=False,connection=False):
   connection = self._getConnection(connection)
   # metadata can be any of ['FileID','Size','UID','GID','Checksum','ChecksumType','Type','CreationDate','ModificationDate','Mode','Status']
   req = "SELECT FileName,%s FROM FC_Files WHERE DirID=%d" % (intListToString(metadata),dirID)
   if not allStatus:
     statusIDs = []
     res = self._getStatusInt('AprioriGood',connection=connection)
     if res['OK']:
       statusIDs.append(res['Value'])
     if statusIDs:
       req = "%s AND Status IN (%s)" % (req,intListToString(statusIDs))
   if fileNames:
     req = "%s AND FileName IN (%s)" % (req,stringListToString(fileNames))
   res = self.db._query(req,connection)
   if not res['OK']:
     return res
   files = {}
   for tuple in res['Value']:
     fileName = tuple[0]
     files[fileName] = dict(zip(metadata,tuple[1:]))
   return S_OK(files)
Пример #45
0
  def __findFullPathDatasets( self, datasets, connection ):
        
    dirDict = self._getDatasetDirectories( datasets )
    failed = {}
    successful = {}
    result = self.db.dtree.findDirs( dirDict.keys() )
    if not result['OK']:
      return result
    directoryIDs = result['Value']

    directoryPaths = {}
    for dirPath in dirDict:
      if not dirPath in directoryIDs:
        for dsName in dirDict[dirPath]:
          dname = '%s/%s' % (dirPath,dsName)
          dname = dname.replace('//','/')
          failed[dname] = 'No such dataset or directory'
      else:
        directoryPaths[directoryIDs[dirPath]] = dirPath    

    wheres = []
    for dirPath in directoryIDs:
      dsNames = dirDict[dirPath]
      dirID = directoryIDs[dirPath]
      wheres.append( "( DirID=%d AND DatasetName IN (%s) )" % ( dirID, stringListToString( dsNames ) ) )

    req = "SELECT DatasetName,DirID,DatasetID FROM FC_MetaDatasets WHERE %s" % " OR ".join( wheres )
    result = self.db._query(req,connection)
    if not result['OK']:
      return result
    for dsName, dirID, dsID in result['Value']:
      dname = '%s/%s' % ( directoryPaths[dirID], dsName )
      dname = dname.replace('//','/')
      successful[dname] = { "DirID": dirID, "DatasetID": dsID } 
  
    for dataset in datasets:
      if not dataset in successful:
        failed[dataset] = "No such dataset"

    return S_OK({"Successful":successful,"Failed":failed})
Пример #46
0
  def getRequestFileStatus( self, requestID, files ):
    """ collect and return all file statuses given :requestID:

    :param int requestID: Requests.RequestID
    :param list files: [ Files.LFN ]
    """
    req = "SELECT DISTINCT `SubRequestID` FROM `SubRequests` WHERE `RequestID`=%d;" % requestID
    res = self._query( req )
    if not res['OK']:
      return res
    subRequests = []
    for subRequestID in res['Value'][0]:
      subRequests.append( subRequestID )
    req = "SELECT `LFN`,`Status` FROM `Files` WHERE `SubRequestID` IN (%s) AND `LFN` in (%s);" %\
        ( intListToString( subRequests ), stringListToString( files ) )
    res = self._query( req )
    if not res['OK']:
      return res
    files = {}
    for lfn, status in res['Value']:
      files[lfn] = status
    return S_OK( files )
Пример #47
0
 def _findDirectories(self,paths,metadata=[]):
   """ Find file ID if it exists for the given list of LFNs """
   startTime = time.time()
   successful = {}
   failed = {}
   req = "SELECT DirName,DirID" 
   if metadata:
     req = "%s,%s" % (req,intListToString(metadata)) 
   req = "%s FROM DirectoryInfo WHERE DirName IN (%s)" % (req,stringListToString(paths))
   res = self.db._query(req)
   if not res['OK']:
     return res
   for tuple in res['Value']:
     dirName = tuple[0]
     dirID = tuple[1]
     metaDict = {'DirID':dirID}
     metaDict.update(dict(zip(metadata,tuple[2:])))
     successful[dirName] = metaDict
   for path in paths:
     if not successful.has_key(path):
       failed[path] = 'No such file or directory'
   return S_OK({"Successful":successful,"Failed":failed})
Пример #48
0
 def getFTSJobList(self, statusList=None, limit=500):
     """ select FTS jobs with statuses in :statusList: """
     statusList = statusList if statusList else list(FTSJob.INITSTATES +
                                                     FTSJob.TRANSSTATES)
     query = "SELECT * FROM `FTSJob` WHERE `Status` IN (%s) ORDER BY `LastUpdate` DESC LIMIT %s;" % (
         stringListToString(statusList), limit)
     trn = self._transaction([query])
     if not trn['OK']:
         self.log.error("getFTSJobList: %s" % trn['Message'])
         return trn
     ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in trn['Value'][query]]
     for ftsJob in ftsJobs:
         query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
         trn = self._transaction(query)
         if not trn['OK']:
             self.log.error("getFTSJobList: %s" % trn['Message'])
             return trn
         ftsFiles = [
             FTSFile(ftsFileDict) for ftsFileDict in trn['Value'][query]
         ]
         for ftsFile in ftsFiles:
             ftsJob.addFile(ftsFile)
     return S_OK(ftsJobs)
Пример #49
0
    def getLFNForGUID(self, guids, connection=False):
        """Returns the lfns matching given guids"""

        connection = self._getConnection(connection)
        if not guids:
            return S_OK({})
        if not isinstance(guids, (list, tuple)):
            guids = [guids]
        req = "SELECT f.FileID, f.FileName, fi.GUID, f.DirID FROM FC_FileInfo fi"
        req += " JOIN FC_Files f on fi.FileID = f.FileID WHERE GUID IN (%s)" % stringListToString(
            guids)
        res = self.db._query(req, connection)
        if not res["OK"]:
            return res

        fileIDguid = {}
        dirIDFileIDs = {}
        fileIDName = {}
        for fileID, fileName, guid, dirID in res["Value"]:
            fileIDguid[fileID] = guid
            dirIDFileIDs.setdefault(dirID, []).append(fileID)
            fileIDName[fileID] = fileName

        res = self.db.dtree.getDirectoryPaths(list(dirIDFileIDs))
        if not res["OK"]:
            return res

        dirIDName = res["Value"]

        guidLFN = dict([(fileIDguid[fId],
                         os.path.join(dirIDName[dId], fileIDName[fId]))
                        for dId in dirIDName for fId in dirIDFileIDs[dId]])
        failedGuid = set(guids) - set(guidLFN)
        failed = dict.fromkeys(failedGuid,
                               "GUID does not exist") if failedGuid else {}

        return S_OK({"Successful": guidLFN, "Failed": failed})
Пример #50
0
 def __findDirs(self,paths,metadata=['DirName']):
   dirs = {}
   req = "SELECT DirID,%s FROM DirectoryInfo WHERE DirName IN (%s)" % (intListToString(metadata),stringListToString(paths))
   res = self.db._query(req)
   if not res['OK']:
     return res
   if not res['Value']:
     return S_OK(dirs)
   for tuple in res['Value']:
     dirID = tuple[0]
     dirs[dirID] = dict(zip(metadata,tuple[1:]))
   return S_OK(dirs)
Пример #51
0
 def getRequestFileStatus( self, requestID, files ):
   req = "SELECT DISTINCT SubRequestID FROM SubRequests WHERE RequestID = %d;" % requestID
   res = self._query( req )
   if not res['OK']:
     return res
   subRequests = []
   for subRequestID in res['Value'][0]:
     subRequests.append( subRequestID )
   req = "SELECT LFN,Status from Files WHERE SubRequestID IN (%s) AND LFN in (%s);" % ( intListToString( subRequests ), stringListToString( files ) )
   res = self._query( req )
   if not res['OK']:
     return res
   files = {}
   for lfn, status in res['Value']:
     files[lfn] = status
   return S_OK( files )
Пример #52
0
 def _getDirectoryFiles(self,dirID,fileNames,metadata_input,allStatus=False,connection=False):
   """ Get the metadata for files in the same directory
   """
   metadata = list(metadata_input)
   
   connection = self._getConnection(connection)
   # metadata can be any of ['FileID','Size','UID','GID','Status','Checksum','CheckSumType','Type','CreationDate','ModificationDate','Mode']
   req = "SELECT FileName,DirID,FileID,Size,UID,GID,Status FROM FC_Files WHERE DirID=%d" % (dirID)
   if not allStatus:
     statusIDs = []
     res = self._getStatusInt('AprioriGood',connection=connection)
     if res['OK']:
       statusIDs.append(res['Value'])
     if statusIDs:
       req = "%s AND Status IN (%s)" % (req,intListToString(statusIDs))
   if fileNames:
     req = "%s AND FileName IN (%s)" % (req,stringListToString(fileNames))
   res = self.db._query(req,connection)
   if not res['OK']:
     return res
   fileNameIDs = res['Value']
   if not fileNameIDs:
     return S_OK({})
   filesDict = {}
   # If we only requested the FileIDs then there is no need to do anything else
   if metadata == ['FileID']:
     for fileName,dirID,fileID,size,uid,gid,status in fileNameIDs:
       filesDict[fileName] = {'FileID':fileID}
     return S_OK(filesDict)
   # Otherwise get the additionally requested metadata from the FC_FileInfo table
   files = {}
   userDict = {}
   groupDict = {}
   for fileName,dirID,fileID,size,uid,gid,status in fileNameIDs:
     filesDict[fileID] = fileName
     files[fileName] = {}
     if 'Size' in metadata:
       files[fileName]['Size'] = size
     if 'DirID' in metadata:
       files[fileName]['DirID'] = dirID
     if 'UID' in metadata:
       files[fileName]['UID'] = uid
       if uid in userDict:
         owner = userDict[uid]
       else:  
         owner = 'unknown'
         result = self.db.ugManager.getUserName(uid)
         if result['OK']:
           owner = result['Value']
         userDict[uid] = owner  
       files[fileName]['Owner'] = owner   
     if 'GID' in metadata:
       files[fileName]['GID'] = gid
       if gid in groupDict:
         group = groupDict[gid]
       else:    
         group = 'unknown'
         result = self.db.ugManager.getGroupName(gid)
         if result['OK']:
           group = result['Value']
         groupDict[gid] = group  
       files[fileName]['OwnerGroup'] = group    
     if 'Status' in metadata:
       files[fileName]['Status'] = status      
   for element in ['FileID','Size','DirID','UID','GID','Status']:
     if element in metadata:
       metadata.remove(element)    
   metadata.append('FileID')
   metadata.reverse()
   req = "SELECT %s FROM FC_FileInfo WHERE FileID IN (%s)" % (intListToString(metadata),intListToString(filesDict.keys()))  
   res = self.db._query(req,connection)
   if not res['OK']:
     return res
   for tuple in res['Value']:
     fileID = tuple[0]
     rowDict = dict(zip(metadata,tuple))
     files[filesDict[fileID]].update(rowDict)
   return S_OK(files)
Пример #53
0
    def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, connection=False):
        """ Get the metadata for files in the same directory
    """
        metadata = list(metadata_input)

        connection = self._getConnection(connection)
        # metadata can be any of ['FileID','Size','UID','GID','Status','Checksum','ChecksumType',
        # 'Type','CreationDate','ModificationDate','Mode']
        req = "SELECT FileName,DirID,FileID,Size,UID,GID,Status FROM FC_Files WHERE DirID=%d" % (dirID)
        if not allStatus:
            statusIDs = []
            for status in self.db.visibleFileStatus:
                res = self._getStatusInt(status, connection=connection)
                if res["OK"]:
                    statusIDs.append(res["Value"])
            if statusIDs:
                req = "%s AND Status IN (%s)" % (req, intListToString(statusIDs))
        if fileNames:
            req = "%s AND FileName IN (%s)" % (req, stringListToString(fileNames))
        res = self.db._query(req, connection)
        if not res["OK"]:
            return res
        fileNameIDs = res["Value"]
        if not fileNameIDs:
            return S_OK({})
        filesDict = {}
        # If we only requested the FileIDs then there is no need to do anything else
        if metadata == ["FileID"]:
            for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
                filesDict[fileName] = {"FileID": fileID}
            return S_OK(filesDict)
        # Otherwise get the additionally requested metadata from the FC_FileInfo table
        files = {}
        userDict = {}
        groupDict = {}
        for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
            filesDict[fileID] = fileName
            files[fileName] = {}
            if "Size" in metadata:
                files[fileName]["Size"] = size
            if "DirID" in metadata:
                files[fileName]["DirID"] = dirID
            if "UID" in metadata:
                files[fileName]["UID"] = uid
                if uid in userDict:
                    owner = userDict[uid]
                else:
                    owner = "unknown"
                    result = self.db.ugManager.getUserName(uid)
                    if result["OK"]:
                        owner = result["Value"]
                    userDict[uid] = owner
                files[fileName]["Owner"] = owner
            if "GID" in metadata:
                files[fileName]["GID"] = gid
                if gid in groupDict:
                    group = groupDict[gid]
                else:
                    group = "unknown"
                    result = self.db.ugManager.getGroupName(gid)
                    if result["OK"]:
                        group = result["Value"]
                    groupDict[gid] = group
                files[fileName]["OwnerGroup"] = group
            if "Status" in metadata:
                files[fileName]["Status"] = self._getIntStatus(status).get("Value", status)
        for element in ["FileID", "Size", "DirID", "UID", "GID", "Status"]:
            if element in metadata:
                metadata.remove(element)
        metadata.append("FileID")
        metadata.reverse()
        req = "SELECT %s FROM FC_FileInfo WHERE FileID IN (%s)" % (
            intListToString(metadata),
            intListToString(filesDict.keys()),
        )
        res = self.db._query(req, connection)
        if not res["OK"]:
            return res
        for tuple_ in res["Value"]:
            fileID = tuple_[0]
            rowDict = dict(zip(metadata, tuple_))
            files[filesDict[fileID]].update(rowDict)
        return S_OK(files)
Пример #54
0
    def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, connection=False):
        """For a given directory, and eventually given file, returns all the desired metadata

        :param int dirID: directory ID
        :param fileNames: the list of filenames, or []
        :param metadata_input: list of desired metadata.
                   It can be anything from (FileName, DirID, FileID, Size, UID, Owner,
                   GID, OwnerGroup, Status, GUID, Checksum, ChecksumType, Type, CreationDate, ModificationDate, Mode)
        :param bool allStatus: if False, only displays the files whose status is in db.visibleFileStatus

        :returns: S_OK(files), where files is a dictionary indexed on filename, and values are dictionary of metadata
        """

        connection = self._getConnection(connection)

        metadata = list(metadata_input)
        if "UID" in metadata:
            metadata.append("Owner")
        if "GID" in metadata:
            metadata.append("OwnerGroup")
        if "FileID" not in metadata:
            metadata.append("FileID")

        # Format the filenames and status to be used in a IN clause in the sotred procedure
        formatedFileNames = stringListToString(fileNames)
        fStatus = stringListToString(self.db.visibleFileStatus)

        specificFiles = True if len(fileNames) else False
        result = self.db.executeStoredProcedureWithCursor(
            "ps_get_all_info_for_files_in_dir", (dirID, specificFiles, formatedFileNames, allStatus, fStatus)
        )

        if not result["OK"]:
            return result

        fieldNames = [
            "FileName",
            "DirID",
            "FileID",
            "Size",
            "UID",
            "Owner",
            "GID",
            "OwnerGroup",
            "Status",
            "GUID",
            "Checksum",
            "ChecksumType",
            "Type",
            "CreationDate",
            "ModificationDate",
            "Mode",
        ]

        rows = result["Value"]
        files = {}

        for row in rows:

            rowDict = dict(zip(fieldNames, row))
            fileName = rowDict["FileName"]
            # Returns only the required metadata
            files[fileName] = dict((key, rowDict.get(key, "Unknown metadata field")) for key in metadata)

        return S_OK(files)
Пример #55
0
    def _getDirectoryFiles(self,
                           dirID,
                           fileNames,
                           metadata_input,
                           allStatus=False,
                           connection=False):
        """ Get the metadata for files in the same directory
    """
        metadata = list(metadata_input)

        connection = self._getConnection(connection)
        # metadata can be any of ['FileID','Size','UID','GID','Status','Checksum','CheckSumType',
        # 'Type','CreationDate','ModificationDate','Mode']
        req = "SELECT FileName,DirID,FileID,Size,UID,GID,Status FROM FC_Files WHERE DirID=%d" % (
            dirID)
        if not allStatus:
            statusIDs = []
            for status in self.db.visibleFileStatus:
                res = self._getStatusInt(status, connection=connection)
                if res['OK']:
                    statusIDs.append(res['Value'])
            if statusIDs:
                req = "%s AND Status IN (%s)" % (req,
                                                 intListToString(statusIDs))
        if fileNames:
            req = "%s AND FileName IN (%s)" % (req,
                                               stringListToString(fileNames))
        res = self.db._query(req, connection)
        if not res['OK']:
            return res
        fileNameIDs = res['Value']
        if not fileNameIDs:
            return S_OK({})
        filesDict = {}
        # If we only requested the FileIDs then there is no need to do anything else
        if metadata == ['FileID']:
            for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
                filesDict[fileName] = {'FileID': fileID}
            return S_OK(filesDict)
        # Otherwise get the additionally requested metadata from the FC_FileInfo table
        files = {}
        userDict = {}
        groupDict = {}
        for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
            filesDict[fileID] = fileName
            files[fileName] = {}
            if 'Size' in metadata:
                files[fileName]['Size'] = size
            if 'DirID' in metadata:
                files[fileName]['DirID'] = dirID
            if 'UID' in metadata:
                files[fileName]['UID'] = uid
                if uid in userDict:
                    owner = userDict[uid]
                else:
                    owner = 'unknown'
                    result = self.db.ugManager.getUserName(uid)
                    if result['OK']:
                        owner = result['Value']
                    userDict[uid] = owner
                files[fileName]['Owner'] = owner
            if 'GID' in metadata:
                files[fileName]['GID'] = gid
                if gid in groupDict:
                    group = groupDict[gid]
                else:
                    group = 'unknown'
                    result = self.db.ugManager.getGroupName(gid)
                    if result['OK']:
                        group = result['Value']
                    groupDict[gid] = group
                files[fileName]['OwnerGroup'] = group
            if 'Status' in metadata:
                files[fileName]['Status'] = status
        for element in ['FileID', 'Size', 'DirID', 'UID', 'GID', 'Status']:
            if element in metadata:
                metadata.remove(element)
        metadata.append('FileID')
        metadata.reverse()
        req = "SELECT %s FROM FC_FileInfo WHERE FileID IN (%s)" % (
            intListToString(metadata), intListToString(filesDict.keys()))
        res = self.db._query(req, connection)
        if not res['OK']:
            return res
        for tuple_ in res['Value']:
            fileID = tuple_[0]
            rowDict = dict(zip(metadata, tuple_))
            files[filesDict[fileID]].update(rowDict)
        return S_OK(files)