def getDirectoryReplicas( self, dirID, path, allStatus = False, connection = False ):
"""
This is defined in the FileManagerBase but it relies on the SEManager to get the SE names.
It is good practice in software, but since the SE and Replica tables are bound together in the DB,
I might as well resolve the name in the query
Get the replicas for all the Files in the given Directory
:param DirID : ID of the directory
:param path : useless
:param allStatus : whether all replicas and file status are considered
If False, take the visibleFileStatus and visibleReplicaStatus values from the configuration
"""
# We format the visible file/replica satus so we can give it as argument to the ps
# It is used in an IN clause, so it looks like --'"AprioriGood","Trash"'--
# fStatus = ','.join( [ '"%s"' % status for status in self.db.visibleFileStatus ] )
# rStatus = ','.join( [ '"%s"' % status for status in self.db.visibleReplicaStatus ] )
fStatus = stringListToString( self.db.visibleFileStatus )
rStatus = stringListToString( self.db.visibleReplicaStatus )
result = self.db.executeStoredProcedureWithCursor( 'ps_get_replicas_for_files_in_dir', ( dirID, allStatus, fStatus, rStatus ) )
if not result['OK']:
return result
resultDict = {}
for fileName, _fileID, seName, pfn in result['Value']:
resultDict.setdefault( fileName, {} ).setdefault( seName, [] ).append( pfn )
return S_OK( resultDict )
def _getDirectoryFiles(self,dirID,fileNames,metadata_input,allStatus=False,connection=False):
""" For a given directory, and eventually given file, returns all the desired metadata
:param dirID : directory ID
:param filenames : the list of filenames, or []
:param metadata_input: list of desired metadata.
It can be anything from (FileName, DirID, FileID, Size, UID, Owner,
GID, OwnerGroup, Status, GUID, Checksum, ChecksumType, Type, CreationDate, ModificationDate, Mode)
:param allStatus : if False, only displays the files whose status is in db.visibleFileStatus
:returns: S_OK(files), where files is a dictionary indexed on filename, and values are dictionary of metadata
"""
connection = self._getConnection( connection )
metadata = list( metadata_input )
if "UID" in metadata:
metadata.append( "Owner" )
if "GID" in metadata:
metadata.append( "OwnerGroup" )
if "FileID" not in metadata:
metadata.append( "FileID" )
# Format the filenames and status to be used in a IN clause in the sotred procedure
formatedFileNames = stringListToString( fileNames )
fStatus = stringListToString( self.db.visibleFileStatus )
specificFiles = True if len( fileNames ) else False
result = self.db.executeStoredProcedureWithCursor( 'ps_get_all_info_for_files_in_dir',
( dirID, specificFiles,
formatedFileNames, allStatus, fStatus ) )
if not result['OK']:
return result
fieldNames = ["FileName", "DirID", "FileID", "Size", "UID", "Owner",
"GID", "OwnerGroup", "Status", "GUID", "Checksum",
"ChecksumType", "Type", "CreationDate", "ModificationDate", "Mode"]
rows = result['Value']
files = {}
for row in rows:
rowDict = dict( zip( fieldNames, row ) )
fileName = rowDict['FileName']
# Returns only the required metadata
files[fileName] = dict( ( key, rowDict.get( key, "Unknown metadata field" ) ) for key in metadata )
return S_OK( files )
def getFTSFileList( self, statusList = None, limit = 1000 ):
""" get at most :limit: FTSFiles with status in :statusList:
:param statusList: list with FTSFiles statuses
:type statusList: python:list
:param int limit: select query limit
"""
statusList = statusList if statusList else [ "Waiting" ]
reStatus = []
inStatus = []
for status in statusList:
if "%" in status or ".*" in status or ".+" in status:
reStatus.append( status )
else:
inStatus.append( status )
reQuery = "`Status` REGEXP '%s'" % "|".join( reStatus ) if reStatus else ""
inQuery = "`Status` IN (%s)" % stringListToString( inStatus ) if inStatus else ""
whereClause = " OR ".join( [ q for q in ( reQuery, inQuery ) if q ] )
if whereClause:
whereClause = "WHERE %s" % whereClause
query = "SELECT * FROM `FTSFile` %s ORDER BY `LastUpdate` DESC LIMIT %s;" % ( whereClause, limit )
trn = self._transaction( [query] )
if not trn['OK']:
self.log.error( "Failed getFTSFileList", "%s" % trn['Message'] )
return trn
return S_OK( [ FTSFile( fileDict ) for fileDict in trn['Value'][query] ] )
def getDatasets( self, datasetName, credDict ):
""" Get information about existing datasets
"""
dsName = os.path.basename(datasetName)
parameterList = ['DatasetID','MetaQuery','DirID','TotalSize','NumberOfFiles',
'UID','GID','Status','CreationDate','ModificationDate',
'DatasetHash','Mode','DatasetName']
parameterString = ','.join( parameterList )
req = "SELECT %s FROM FC_MetaDatasets" % parameterString
if type( datasetName ) in StringTypes:
dsName = os.path.basename(datasetName)
if '*' in dsName:
dName = dsName.replace( '*', '%' )
req += " WHERE DatasetName LIKE '%s'" % dName
elif dsName:
req += " WHERE DatasetName='%s'" % dsName
elif type( datasetName ) == ListType:
dsNames = [ os.path.basename(d) for d in datasetName ]
datasetString = stringListToString( dsNames )
req += " WHERE DatasetName in (%s)" % datasetString
result = self.db._query( req )
if not result['OK']:
return result
resultDict = {}
for row in result['Value']:
dName = row[12]
resultDict[dName] = self.__getDatasetDict( row )
return S_OK( resultDict )
def _getFileIDFromGUID( self, guids, connection = False ):
""" Returns the file ids from list of guids
:param guids : list of guid
:returns dictionary < guid : fileId >
"""
connection = self._getConnection(connection)
if not guids:
return S_OK({})
if type( guids ) not in [ListType, TupleType]:
guids = [guids]
# formatedGuids = ','.join( [ '"%s"' % guid for guid in guids ] )
formatedGuids = stringListToString( guids )
result = self.db.executeStoredProcedureWithCursor( 'ps_get_file_ids_from_guids', ( formatedGuids, ) )
if not result['OK']:
return result
guidDict = dict( ( guid, fileID ) for guid, fileID in result['Value'] )
return S_OK(guidDict)
def _checkTaskUpdate( self, taskIDs, newTaskState, connection = False ):
connection = self.__getConnection( connection )
if not taskIDs:
return S_OK( taskIDs )
# * -> Failed
if newTaskState == 'Failed':
oldTaskState = []
# StageCompleting -> Done
elif newTaskState == 'Done':
oldTaskState = ['StageCompleting']
# StageSubmitted -> StageCompleting
elif newTaskState == 'StageCompleting':
oldTaskState = ['StageSubmitted']
# Waiting -> StageSubmitted
elif newTaskState == 'StageSubmitted':
oldTaskState = ['Waiting', 'Offline']
# New -> Waiting
elif newTaskState == 'Waiting':
oldTaskState = ['New']
elif newTaskState == 'Offline':
oldTaskState = ['Waiting']
else:
return S_ERROR( "Task status not recognized" )
if not oldTaskState:
toUpdate = taskIDs
else:
req = "SELECT TaskID FROM Tasks WHERE Status in (%s) AND TaskID IN (%s)" % ( stringListToString( oldTaskState ), intListToString( taskIDs ) )
res = self._query( req, connection )
if not res['OK']:
return res
toUpdate = [row[0] for row in res['Value']]
return S_OK( toUpdate )
def __findNoPathDatasets( self, nodirDatasets, connection ):
failed = {}
successful = {}
dsIDs = {}
req = "SELECT COUNT(DatasetName),DatasetName,DatasetID FROM FC_MetaDatasets WHERE DatasetName in "
req += "( %s ) GROUP BY DatasetName" % stringListToString( nodirDatasets )
result = self.db._query( req, connection )
if not result['OK']:
return result
for dsCount, dsName, dsID in result['Value']:
if dsCount > 1:
failed[dsName] = "Ambiguous dataset name"
else:
dsIDs[dsName] = str( dsID )
if dsIDs:
req = "SELECT DatasetName,DatasetID,DirID FROM FC_MetaDatasets WHERE DatasetID in (%s)" % ','.join( dsIDs.values() )
result = self.db._query( req, connection )
if not result['OK']:
return result
for dsName, dsID, dirID in result['Value']:
successful[dsName] = { "DirID": dirID, "DatasetID": dsID }
for name in nodirDatasets:
if name not in failed and name not in successful:
failed[name] = "Dataset not found"
return S_OK({"Successful":successful,"Failed":failed})
def getFTSJobsForRequest( self, requestID, statusList = None ):
""" get list of FTSJobs with status in :statusList: for request given its requestID
TODO: should be more smart, i.e. one query to select all ftsfiles
"""
statusList = statusList if statusList else list( FTSJob.INITSTATES + FTSJob.TRANSSTATES )
query = "SELECT * FROM `FTSJob` WHERE `RequestID` = %s AND `Status` in (%s)" % ( requestID,
stringListToString( statusList ) )
ftsJobs = self._transaction( [ query ] )
if not ftsJobs['OK']:
self.log.error( "Failed getFTSJobsForRequest", "%s" % ftsJobs['Message'] )
return ftsJobs
ftsJobs = ftsJobs['Value'][query] if query in ftsJobs['Value'] else []
ftsJobs = [ FTSJob( ftsJobDict ) for ftsJobDict in ftsJobs ]
for ftsJob in ftsJobs:
query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s' AND `RequestID`=%s" % ( ftsJob.FTSGUID,
requestID )
ftsFiles = self._transaction( [ query ] )
if not ftsFiles['OK']:
self.log.error( "Failed getFTSJobsForRequest", "%s" % ftsFiles['Message'] )
return ftsFiles
ftsFiles = ftsFiles['Value'][query] if query in ftsFiles['Value'] else []
for ftsFileDict in ftsFiles:
ftsJob.addFile( FTSFile( ftsFileDict ) )
return S_OK( ftsJobs )
def getFTSFileIDs( self, statusList = None ):
""" select FTSFileIDs for a given status list """
statusList = statusList if statusList else [ "Waiting" ]
query = "SELECT `FTSFileID` FROM `FTSFile` WHERE `Status` IN (%s);" % stringListToString( statusList );
query = self._query( query )
if not query['OK']:
self.log.error( 'Failed ftsFileSQL', query['Message'] )
return query
return S_OK( [ item[0] for item in query['Value'] ] )
def getFTSJobIDs( self, statusList = [ "Submitted", "Active", "Ready" ] ):
""" get FTSJobIDs for a given status list """
query = "SELECT `FTSJobID` FROM `FTSJob` WHERE `Status` IN (%s);" % stringListToString( statusList )
query = self._query( query )
if not query['OK']:
self.log.error( 'Failed ftsJobSQL', query['Message'] )
return query
# # convert to list of longs
return S_OK( [ item[0] for item in query['Value'] ] )
def getFTSJobIDs(self, statusList=["Submitted", "Active", "Ready"]):
""" get FTSJobIDs for a given status list """
query = "SELECT `FTSJobID` FROM `FTSJob` WHERE `Status` IN (%s);" % stringListToString(
statusList)
query = self._query(query)
if not query['OK']:
self.log.error('Failed ftsJobSQL', query['Message'])
return query
# # convert to list of longs
return S_OK([item[0] for item in query['Value']])
def getFTSFileIDs(self, statusList=None):
""" select FTSFileIDs for a given status list """
statusList = statusList if statusList else ["Waiting"]
query = "SELECT `FTSFileID` FROM `FTSFile` WHERE `Status` IN (%s);" % stringListToString(
statusList)
query = self._query(query)
if not query['OK']:
self.log.error('Failed ftsFileSQL', query['Message'])
return query
return S_OK([item[0] for item in query['Value']])
def _findFileIDs(self, lfns, connection=False):
"""Find lfn <-> FileID correspondence"""
connection = self._getConnection(connection)
failed = {}
successful = {}
# If there is only one lfn, we might as well make a direct query
if len(lfns) == 1:
lfn = list(lfns)[0] # if lfns is a dict, list(lfns) returns lfns.keys()
pathPart, filePart = os.path.split(lfn)
result = self.db.executeStoredProcedure(
"ps_get_file_id_from_lfn", (pathPart, filePart, "ret1"), outputIds=[2]
)
if not result["OK"]:
return result
fileId = result["Value"][0]
if not fileId:
failed[lfn] = "No such file or directory"
else:
successful[lfn] = fileId
else:
# We separate the files by directory
filesInDirDict = self._getFileDirectories(lfns)
# We get the directory ids
result = self.db.dtree.findDirs(list(filesInDirDict))
if not result["OK"]:
return result
directoryPathToIds = result["Value"]
# For each directory, we get the file ids of the files we want
for dirPath in directoryPathToIds:
fileNames = filesInDirDict[dirPath]
dirID = directoryPathToIds[dirPath]
formatedFileNames = stringListToString(fileNames)
result = self.db.executeStoredProcedureWithCursor(
"ps_get_file_ids_from_dir_id", (dirID, formatedFileNames)
)
if not result["OK"]:
return result
for fileID, fileName in result["Value"]:
fname = os.path.join(dirPath, fileName)
successful[fname] = fileID
# The lfns that are not in successful dont exist
for failedLfn in set(lfns) - set(successful):
failed[failedLfn] = "No such file or directory"
return S_OK({"Successful": successful, "Failed": failed})
def _getFileReplicas(self,
fileIDs,
fields_input=['PFN'],
allStatus=False,
connection=False):
""" Get replicas for the given list of files specified by their fileIDs
:param fileIDs : list of file ids
:param fields_input : metadata of the Replicas we are interested in
:param allStatus : if True, all the Replica statuses will be considered, otherwise, only the db.visibleReplicaStatus
:returns S_OK with a dict { fileID : { SE name : dict of metadata } }
"""
connection = self._getConnection(connection)
fields = list(fields_input)
if 'Status' not in fields:
fields.append('Status')
replicas = {}
# Format the status to be used in a IN clause in the stored procedure
fStatus = stringListToString(self.db.visibleReplicaStatus)
fieldNames = [
"FileID", "SE", "Status", "RepType", "CreationDate",
"ModificationDate", "PFN"
]
for fileID in fileIDs:
result = self.db.executeStoredProcedureWithCursor(
'ps_get_all_info_of_replicas', (fileID, allStatus, fStatus))
if not result['OK']:
return result
rows = result['Value']
if not rows:
replicas[fileID] = {}
for row in rows:
rowDict = dict(zip(fieldNames, row))
# Returns only the required metadata
se = rowDict["SE"]
repForFile = replicas.setdefault(fileID, {})
repForFile[se] = dict(
(key, rowDict.get(key, "Unknown metadata field"))
for key in fields)
return S_OK(replicas)
def removeMigratingFiles(self,lfns):
""" Remove all replicas with associated LFNs from the database
"""
gLogger.info("removeMigratingFiles: Attempting to remove %d files from the database." % len(lfns))
req = "DELETE FROM MigratingReplicas WHERE LFN IN (%s);" % stringListToString(lfns)
res = self._update(req)
if not res['OK']:
gLogger.error("removeMigratingFiles: Failed removing files from database.",res['Message'])
else:
gLogger.info("removeMigratingFiles: Successfully removed files.")
return res
def _findFileIDs(self, lfns, connection=False):
""" Find lfn <-> FileID correspondence
"""
connection = self._getConnection(connection)
failed = {}
successful = {}
# If there is only one lfn, we might as well make a direct query
if len(lfns) == 1:
lfn = list(lfns)[0] # if lfns is a dict, list(lfns) returns lfns.keys()
pathPart, filePart = os.path.split(lfn)
result = self.db.executeStoredProcedure(
'ps_get_file_id_from_lfn', (pathPart, filePart, 'ret1'), outputIds=[2])
if not result['OK']:
return result
fileId = result['Value'][0]
if not fileId:
failed[lfn] = "No such file or directory"
else:
successful[lfn] = fileId
else:
# We separate the files by directory
filesInDirDict = self._getFileDirectories(lfns)
# We get the directory ids
result = self.db.dtree.findDirs(filesInDirDict.keys())
if not result['OK']:
return result
directoryPathToIds = result['Value']
# For each directory, we get the file ids of the files we want
for dirPath in directoryPathToIds:
fileNames = filesInDirDict[dirPath]
dirID = directoryPathToIds[dirPath]
formatedFileNames = stringListToString(fileNames)
result = self.db.executeStoredProcedureWithCursor(
'ps_get_file_ids_from_dir_id', (dirID, formatedFileNames))
if not result['OK']:
return result
for fileID, fileName in result['Value']:
fname = os.path.join(dirPath, fileName)
successful[fname] = fileID
# The lfns that are not in successful dont exist
for failedLfn in (set(lfns) - set(successful)):
failed[failedLfn] = "No such file or directory"
return S_OK({"Successful": successful, "Failed": failed})
def getRequestNamesList( self, statusList = None, limit = None ):
""" select requests with status in :statusList: """
statusList = statusList if statusList else list( Request.FINAL_STATES )
limit = limit if limit else 100
query = "SELECT `RequestName`, `Status`, `LastUpdate` FROM `Request` WHERE "\
" `Status` IN (%s) ORDER BY `LastUpdate` DESC LIMIT %s;" % ( stringListToString( statusList ), limit )
reqNamesList = self._query( query )
if not reqNamesList["OK"]:
self.log.error( "getRequestNamesList: %s" % reqNamesList["Message"] )
return reqNamesList
reqNamesList = reqNamesList["Value"]
return S_OK( [ reqName for reqName in reqNamesList] )
def getFTSFilesForRequest(self, requestID, statusList=None):
""" get FTSFiles with status in :statusList: for request given its :requestID: """
requestID = int(requestID)
statusList = statusList if statusList else ["Waiting"]
query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s AND `Status` IN (%s);" % (
requestID, stringListToString(statusList))
ftsFiles = self._transaction([query])
if not ftsFiles['OK']:
self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message'])
return ftsFiles
ftsFiles = ftsFiles['Value'].get(query, [])
return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
def getDirectoryReplicas(self,
dirID,
path,
allStatus=False,
connection=False):
"""
This is defined in the FileManagerBase but it relies on the SEManager to get the SE names.
It is good practice in software, but since the SE and Replica tables are bound together in the DB,
I might as well resolve the name in the query
Get the replicas for all the Files in the given Directory
:param int dirID: ID of the directory
:param unused path: useless
:param bool allStatus: whether all replicas and file status are considered
If False, take the visibleFileStatus and visibleReplicaStatus
values from the configuration
"""
# We format the visible file/replica satus so we can give it as argument to the ps
# It is used in an IN clause, so it looks like --'"AprioriGood","Trash"'--
# fStatus = ','.join( [ '"%s"' % status for status in self.db.visibleFileStatus ] )
# rStatus = ','.join( [ '"%s"' % status for status in self.db.visibleReplicaStatus ] )
fStatus = stringListToString(self.db.visibleFileStatus)
rStatus = stringListToString(self.db.visibleReplicaStatus)
result = self.db.executeStoredProcedureWithCursor(
"ps_get_replicas_for_files_in_dir",
(dirID, allStatus, fStatus, rStatus))
if not result["OK"]:
return result
resultDict = {}
for fileName, _fileID, seName, pfn in result["Value"]:
resultDict.setdefault(fileName, {}).setdefault(seName,
[]).append(pfn)
return S_OK(resultDict)
def __findDirs(self, paths, metadata=['DirName']):
dirs = {}
req = "SELECT DirID,%s FROM DirectoryInfo WHERE DirName IN (%s)" % (
intListToString(metadata), stringListToString(paths))
res = self.db._query(req)
if not res['OK']:
return res
if not res['Value']:
return S_OK(dirs)
for tup in res['Value']:
dirID = tup[0]
dirs[dirID] = dict(zip(metadata, tup[1:]))
return S_OK(dirs)
def _getFileIDFromGUID(self, guid, connection=False):
connection = self._getConnection(connection)
if not guid:
return S_OK({})
if not isinstance(guid, (list, tuple)):
guid = [guid]
req = "SELECT FileID,GUID FROM FC_Files WHERE GUID IN (%s)" % stringListToString(guid)
res = self.db._query(req, connection)
if not res['OK']:
return res
guidDict = {}
for fileID, guid in res['Value']:
guidDict[guid] = fileID
return S_OK(guidDict)
def getRequestNamesList( self, statusList = None, limit = None, since = None, until = None ):
""" select requests with status in :statusList: """
statusList = statusList if statusList else list( Request.FINAL_STATES )
limit = limit if limit else 100
sinceReq = " AND LastUpdate > %s " % since if since else ""
untilReq = " AND LastUpdate < %s " % until if until else ""
query = "SELECT `RequestName`, `Status`, `LastUpdate` FROM `Request` WHERE "\
" `Status` IN (%s) %s %s ORDER BY `LastUpdate` ASC LIMIT %s;" % ( stringListToString( statusList ), sinceReq, untilReq, limit )
reqNamesList = self._query( query )
if not reqNamesList["OK"]:
self.log.error( "getRequestNamesList: %s" % reqNamesList["Message"] )
return reqNamesList
reqNamesList = reqNamesList["Value"]
return S_OK( [ reqName for reqName in reqNamesList] )
def _getFileIDFromGUID(self,guid,connection=False):
connection = self._getConnection(connection)
if not guid:
return S_OK({})
if type(guid) not in [ListType,TupleType]:
guid = [guid]
req = "SELECT FileID,GUID FROM FC_FileInfo WHERE GUID IN (%s)" % stringListToString(guid)
res = self.db._query(req,connection)
if not res['OK']:
return res
guidDict = {}
for fileID,guid in res['Value']:
guidDict[guid] = fileID
return S_OK(guidDict)
def _getFileReplicas( self, fileIDs, fields_input = ['PFN'], allStatus = False, connection = False ):
""" Get replicas for the given list of files specified by their fileIDs
:param fileIDs : list of file ids
:param fields_input : metadata of the Replicas we are interested in
:param allStatus : if True, all the Replica statuses will be considered, otherwise, only the db.visibleReplicaStatus
:returns S_OK with a dict { fileID : { SE name : dict of metadata } }
"""
connection = self._getConnection( connection )
fields = list( fields_input )
if 'Status' not in fields:
fields.append( 'Status' )
replicas = {}
# Format the status to be used in a IN clause in the stored procedure
fStatus = stringListToString( self.db.visibleReplicaStatus )
fieldNames = [ "FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"]
for fileID in fileIDs:
result = self.db.executeStoredProcedureWithCursor( 'ps_get_all_info_of_replicas',
( fileID, allStatus, fStatus ) )
if not result['OK']:
return result
rows = result['Value']
if not rows:
replicas[fileID] = {}
for row in rows:
rowDict = dict( zip( fieldNames, row ) )
# Returns only the required metadata
se = rowDict["SE"]
repForFile = replicas.setdefault( fileID, {} )
repForFile[se] = dict( ( key, rowDict.get( key, "Unknown metadata field" ) ) for key in fields )
return S_OK(replicas)
def _getFileReplicas(self, fileIDs, fields_input=None, allStatus=False, connection=False):
"""Get replicas for the given list of files specified by their fileIDs
:param fileIDs : list of file ids
:param fields_input : metadata of the Replicas we are interested in (default to PFN)
:param allStatus : if True, all the Replica statuses will be considered,
otherwise, only the db.visibleReplicaStatus
:returns S_OK with a dict { fileID : { SE name : dict of metadata } }
"""
if fields_input is None:
fields_input = ["PFN"]
fields = list(fields_input)
# always add Status in the list of required fields
if "Status" not in fields:
fields.append("Status")
# We initialize the dictionary with empty dict
# as default value, because this is what we want for
# non existing replicas
replicas = {fileID: {} for fileID in fileIDs}
# Format the status to be used in a IN clause in the stored procedure
fStatus = stringListToString(self.db.visibleReplicaStatus)
fieldNames = ["FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"]
for chunks in breakListIntoChunks(fileIDs, 1000):
# Format the FileIDs to be used in a IN clause in the stored procedure
formatedFileIds = intListToString(chunks)
result = self.db.executeStoredProcedureWithCursor(
"ps_get_all_info_of_replicas_bulk", (formatedFileIds, allStatus, fStatus)
)
if not result["OK"]:
return result
rows = result["Value"]
for row in rows:
rowDict = dict(zip(fieldNames, row))
se = rowDict["SE"]
fileID = rowDict["FileID"]
replicas[fileID][se] = dict((key, rowDict.get(key, "Unknown metadata field")) for key in fields)
return S_OK(replicas)
def __deleteFiles(self, fileIDs, connection=False):
connection = self._getConnection(connection)
if type(fileIDs) not in [ListType, TupleType]:
fileIDs = [fileIDs]
if not fileIDs:
return S_OK()
fileIDString = intListToString(fileIDs)
failed = []
for table in ["FC_Files", "FC_FileInfo"]:
req = "DELETE FROM %s WHERE FileID in (%s)" % (table, fileIDString)
res = self.db._update(req, connection)
if not res["OK"]:
gLogger.error("Failed to remove files from table %s" % table, res["Message"])
failed.append(table)
if failed:
return S_ERROR("Failed to remove files from %s" % stringListToString(failed))
return S_OK()
def getRequestFileStatus(self, requestID, files):
req = "SELECT DISTINCT SubRequestID FROM SubRequests WHERE RequestID = %d;" % requestID
res = self._query(req)
if not res['OK']:
return res
subRequests = []
for subRequestID in res['Value'][0]:
subRequests.append(subRequestID)
req = "SELECT LFN,Status from Files WHERE SubRequestID IN (%s) AND LFN in (%s);" % (
intListToString(subRequests), stringListToString(files))
res = self._query(req)
if not res['OK']:
return res
files = {}
for lfn, status in res['Value']:
files[lfn] = status
return S_OK(files)
def __deleteFiles( self, fileIDs, connection = False ):
connection = self._getConnection(connection)
if type(fileIDs) not in [ ListType, TupleType]:
fileIDs = [fileIDs]
if not fileIDs:
return S_OK()
fileIDString = intListToString( fileIDs )
failed = []
for table in ['FC_Files','FC_FileInfo']:
req = "DELETE FROM %s WHERE FileID in (%s)" % ( table, fileIDString )
res = self.db._update(req,connection)
if not res['OK']:
gLogger.error( "Failed to remove files from table %s" % table, res['Message'] )
failed.append(table)
if failed:
return S_ERROR( "Failed to remove files from %s" % stringListToString( failed ) )
return S_OK()
def __deleteReplicas(self,repIDs,connection=False):
connection = self._getConnection(connection)
if type(repIDs) not in [ ListType, TupleType]:
repIDs = [repIDs]
if not repIDs:
return S_OK()
repIDString = intListToString(repIDs)
failed = []
for table in ['FC_Replicas','FC_ReplicaInfo']:
req = "DELETE FROM %s WHERE RepID in (%s)" % (table,repIDString)
res = self.db._update(req,connection)
if not res['OK']:
gLogger.error("Failed to remove replicas from %s" % table,res['Message'])
failed.append(table)
if failed:
return S_ERROR("Failed to remove replicas from %s" % stringListToString(failed))
return S_OK()
def __deleteReplicas(self, repIDs, connection=False):
connection = self._getConnection(connection)
if not isinstance(repIDs, (list, tuple)):
repIDs = [repIDs]
if not repIDs:
return S_OK()
repIDString = intListToString(repIDs)
failed = []
for table in ['FC_Replicas', 'FC_ReplicaInfo']:
req = "DELETE FROM %s WHERE RepID in (%s)" % (table, repIDString)
res = self.db._update(req, connection)
if not res['OK']:
gLogger.error("Failed to remove replicas from table %s" % table, res['Message'])
failed.append(table)
if failed:
return S_ERROR("Failed to remove replicas from %s" % stringListToString(failed))
return S_OK()
def _findFileIDs(self, lfns, connection=False):
""" Find lfn <-> FileID correspondence
"""
connection = self._getConnection(connection)
dirDict = self._getFileDirectories(lfns)
failed = {}
successful = {}
result = self.db.dtree.findDirs(dirDict.keys())
if not result['OK']:
return result
directoryIDs = result['Value']
directoryPaths = {}
for dirPath in dirDict:
if not dirPath in directoryIDs:
for fileName in dirDict[dirPath]:
fname = '%s/%s' % (dirPath, fileName)
fname = fname.replace('//', '/')
failed[fname] = 'No such directory'
else:
directoryPaths[directoryIDs[dirPath]] = dirPath
directoryIDList = directoryIDs.keys()
for dirIDs in breakListIntoChunks(directoryIDList, 1000):
wheres = []
for dirPath in dirIDs:
fileNames = dirDict[dirPath]
dirID = directoryIDs[dirPath]
wheres.append("( DirID=%d AND FileName IN (%s) )" %
(dirID, stringListToString(fileNames)))
req = "SELECT FileName,DirID,FileID FROM FC_Files WHERE %s" % " OR ".join(
wheres)
result = self.db._query(req, connection)
if not result['OK']:
return result
for fileName, dirID, fileID in result['Value']:
fname = '%s/%s' % (directoryPaths[dirID], fileName)
fname = fname.replace('//', '/')
successful[fname] = fileID
for lfn in lfns:
if not lfn in successful:
failed[lfn] = "No such file"
return S_OK({"Successful": successful, "Failed": failed})
def findDirs( self, paths, connection = False ):
""" Find DirIDs for the given path list
:param paths list of path
:returns S_OK( { path : ID} )
"""
dirDict = {}
if not paths:
return S_OK( dirDict )
dpaths = stringListToString( [os.path.normpath( path ) for path in paths ] )
result = self.db.executeStoredProcedureWithCursor( 'ps_find_dirs', ( dpaths, ) )
if not result['OK']:
return result
for dirName, dirID in result['Value']:
dirDict[dirName] = dirID
return S_OK( dirDict )
def getRequestNamesList(self,
statusList=None,
limit=None,
since=None,
until=None):
""" select requests with status in :statusList: """
statusList = statusList if statusList else list(Request.FINAL_STATES)
limit = limit if limit else 100
sinceReq = " AND LastUpdate > %s " % since if since else ""
untilReq = " AND LastUpdate < %s " % until if until else ""
query = "SELECT `RequestName`, `Status`, `LastUpdate` FROM `Request` WHERE "\
" `Status` IN (%s) %s %s ORDER BY `LastUpdate` ASC LIMIT %s;" % ( stringListToString( statusList ), sinceReq, untilReq, limit )
reqNamesList = self._query(query)
if not reqNamesList["OK"]:
self.log.error("getRequestNamesList: %s" % reqNamesList["Message"])
return reqNamesList
reqNamesList = reqNamesList["Value"]
return S_OK([reqName for reqName in reqNamesList])
def _findFileIDs( self, lfns, connection=False ):
""" Find lfn <-> FileID correspondence
"""
connection = self._getConnection(connection)
dirDict = self._getFileDirectories(lfns)
failed = {}
successful = {}
result = self.db.dtree.findDirs( dirDict.keys() )
if not result['OK']:
return result
directoryIDs = result['Value']
directoryPaths = {}
for dirPath in dirDict:
if not dirPath in directoryIDs:
for fileName in dirDict[dirPath]:
fname = '%s/%s' % (dirPath,fileName)
fname = fname.replace('//','/')
failed[fname] = 'No such file or directory'
else:
directoryPaths[directoryIDs[dirPath]] = dirPath
directoryIDList = directoryIDs.keys()
for dirIDs in breakListIntoChunks( directoryIDList, 1000 ):
wheres = []
for dirPath in dirIDs:
fileNames = dirDict[dirPath]
dirID = directoryIDs[dirPath]
wheres.append( "( DirID=%d AND FileName IN (%s) )" % (dirID, stringListToString(fileNames) ) )
req = "SELECT FileName,DirID,FileID FROM FC_Files WHERE %s" % " OR ".join( wheres )
result = self.db._query(req,connection)
if not result['OK']:
return result
for fileName, dirID, fileID in result['Value']:
fname = '%s/%s' % (directoryPaths[dirID],fileName)
fname = fname.replace('//','/')
successful[fname] = fileID
for lfn in lfns:
if not lfn in successful:
failed[lfn] = "No such file or directory"
return S_OK({"Successful":successful,"Failed":failed})
def getFiles(self, status):
""" Obtain all the files in a give status (or list) in the database along with all their associated metadata
:param status: string or list of status we want to query
:returns: dict {lfn: metadata}
"""
if isinstance(status, basestring):
status = [status]
try:
gLogger.info(
"RAWIntegrityDB.getActiveFiles: Obtaining files in status %s."
% status)
req = "SELECT LFN,Status,PFN,Size,StorageElement,GUID,FileChecksum,TIME_TO_SEC(TIMEDIFF(UTC_TIMESTAMP(),SubmitTime)),SubmitTime from Files WHERE Status in (%s);" % (
stringListToString(status))
res = self._query(req)
if not res['OK']:
gLogger.error(
"RAWIntegrityDB.getActiveFiles: Failed to get files from database.",
res['Message'])
return res
fileDict = {}
for lfn, st, pfn, size, se, guid, checksum, waittime, submitTime in res[
'Value']:
fileDict[lfn] = {
'PFN': pfn,
'Size': size,
'SE': se,
'GUID': guid,
'Checksum': checksum,
'WaitTime': waittime,
'Status': st,
'SubmitTime': submitTime
}
gLogger.info(
"RAWIntegrityDB.getActiveFiles: Obtained %s files awaiting migration from database."
% len(fileDict.keys()))
return S_OK(fileDict)
except Exception as x:
errStr = "RAWIntegrityDB.getActiveFiles: Exception while getting files from database."
gLogger.exception(errStr, lException=x)
return S_ERROR(errStr)
def getLFNForGUID(self, guids, connection=False):
""" Returns the lfns matching given guids"""
connection = self._getConnection(connection)
if not guids:
return S_OK({})
if not isinstance(guids, (list, tuple)):
guids = [guids]
formatedGuids = stringListToString(guids)
result = self.db.executeStoredProcedureWithCursor('ps_get_lfns_from_guids', (formatedGuids, ))
if not result['OK']:
return result
guidDict = dict((guid, lfn) for guid, lfn in result['Value'])
failedGuid = set(guids) - set(guidDict)
failed = dict.fromkeys(failedGuid, "GUID does not exist") if failedGuid else {}
return S_OK({"Successful": guidDict, "Failed": failed})
def findDirs(self, paths, connection=False):
""" Find DirIDs for the given path list
:param paths list of path
:returns S_OK( { path : ID} )
"""
dirDict = {}
if not paths:
return S_OK(dirDict)
dpaths = stringListToString([os.path.normpath(path) for path in paths])
result = self.db.executeStoredProcedureWithCursor(
'ps_find_dirs', (dpaths, ))
if not result['OK']:
return result
for dirName, dirID in result['Value']:
dirDict[dirName] = dirID
return S_OK(dirDict)
def __deleteFiles(self, fileIDs, connection=False):
connection = self._getConnection(connection)
if not isinstance(fileIDs, (list, tuple)):
fileIDs = [fileIDs]
if not fileIDs:
return S_OK()
fileIDString = intListToString(fileIDs)
failed = []
for table in ["FC_Files", "FC_FileInfo"]:
req = "DELETE FROM %s WHERE FileID in (%s)" % (table, fileIDString)
res = self.db._update(req, connection)
if not res["OK"]:
gLogger.error("Failed to remove files from table %s" % table,
res["Message"])
failed.append(table)
if failed:
return S_ERROR("Failed to remove files from %s" %
stringListToString(failed))
return S_OK()
def getLFNForGUID( self, guids, connection = False ):
""" Returns the lfns matching given guids"""
connection = self._getConnection( connection )
if not guids:
return S_OK( {} )
if type( guids ) not in [ListType, TupleType]:
guids = [guids]
formatedGuids = stringListToString( guids )
result = self.db.executeStoredProcedureWithCursor( 'ps_get_lfns_from_guids', ( formatedGuids, ) )
if not result['OK']:
return result
guidDict = dict( ( guid, lfn ) for guid, lfn in result['Value'] )
failedGuid = set( guids ) - set( guidDict )
failed = dict.fromkeys( failedGuid, "GUID does not exist" ) if failedGuid else {}
return S_OK( {"Successful" : guidDict, "Failed" : failed} )
def __findFullPathDatasets(self, datasets, connection):
dirDict = self._getDatasetDirectories(datasets)
failed = {}
successful = {}
result = self.db.dtree.findDirs(list(dirDict))
if not result["OK"]:
return result
directoryIDs = result["Value"]
directoryPaths = {}
for dirPath in dirDict:
if dirPath not in directoryIDs:
for dsName in dirDict[dirPath]:
dname = "%s/%s" % (dirPath, dsName)
dname = dname.replace("//", "/")
failed[dname] = "No such dataset or directory"
else:
directoryPaths[directoryIDs[dirPath]] = dirPath
wheres = []
for dirPath in directoryIDs:
dsNames = dirDict[dirPath]
dirID = directoryIDs[dirPath]
wheres.append("( DirID=%d AND DatasetName IN (%s) )" %
(dirID, stringListToString(dsNames)))
req = "SELECT DatasetName,DirID,DatasetID FROM FC_MetaDatasets WHERE %s" % " OR ".join(
wheres)
result = self.db._query(req, connection)
if not result["OK"]:
return result
for dsName, dirID, dsID in result["Value"]:
dname = "%s/%s" % (directoryPaths[dirID], dsName)
dname = dname.replace("//", "/")
successful[dname] = {"DirID": dirID, "DatasetID": dsID}
for dataset in datasets:
if dataset not in successful:
failed[dataset] = "No such dataset"
return S_OK({"Successful": successful, "Failed": failed})
def getLFNForGUID(self, guids, connection=False):
""" Returns the lfns matching given guids"""
connection = self._getConnection(connection)
if not guids:
return S_OK({})
if type(guids) not in [ListType, TupleType]:
guids = [guids]
req = (
"SELECT f.FileID, f.FileName, fi.GUID, f.DirID FROM FC_FileInfo fi JOIN FC_Files f on fi.FileID = f.FileID WHERE GUID IN (%s)"
% stringListToString(guids)
)
res = self.db._query(req, connection)
if not res["OK"]:
return res
fileIDguid = {}
dirIDFileIDs = {}
fileIDName = {}
for fileID, fileName, guid, dirID in res["Value"]:
fileIDguid[fileID] = guid
dirIDFileIDs.setdefault(dirID, []).append(fileID)
fileIDName[fileID] = fileName
res = self.db.dtree.getDirectoryPaths(dirIDFileIDs.keys())
if not res["OK"]:
return res
dirIDName = res["Value"]
guidLFN = dict(
[
(fileIDguid[fId], os.path.join(dirIDName[dId], fileIDName[fId]))
for dId in dirIDName
for fId in dirIDFileIDs[dId]
]
)
failedGuid = set(guids) - set(guidLFN)
failed = dict.fromkeys(failedGuid, "GUID does not exist") if failedGuid else {}
return S_OK({"Successful": guidLFN, "Failed": failed})
def _findDirectories(self, paths, metadata=[]):
""" Find file ID if it exists for the given list of LFNs """
successful = {}
failed = {}
req = "SELECT DirName,DirID"
if metadata:
req = "%s,%s" % (req, intListToString(metadata))
req = "%s FROM DirectoryInfo WHERE DirName IN (%s)" % (req, stringListToString(paths))
res = self.db._query(req)
if not res['OK']:
return res
for tup in res['Value']:
dirName = tup[0]
dirID = tup[1]
metaDict = {'DirID': dirID}
metaDict.update(dict(zip(metadata, tup[2:])))
successful[dirName] = metaDict
for path in paths:
if path not in successful:
failed[path] = 'No such file or directory'
return S_OK({"Successful": successful, "Failed": failed})
def _getDirectoryFiles(self, dirID, fileNames, metadata, allStatus=False, connection=False):
connection = self._getConnection(connection)
# metadata can be any of ['FileID','Size','UID','GID','Checksum','ChecksumType','Type','CreationDate','ModificationDate','Mode','Status']
req = "SELECT FileName,%s FROM FC_Files WHERE DirID=%d" % (intListToString(metadata), dirID)
if not allStatus:
statusIDs = []
res = self._getStatusInt("AprioriGood", connection=connection)
if res["OK"]:
statusIDs.append(res["Value"])
if statusIDs:
req = "%s AND Status IN (%s)" % (req, intListToString(statusIDs))
if fileNames:
req = "%s AND FileName IN (%s)" % (req, stringListToString(fileNames))
res = self.db._query(req, connection)
if not res["OK"]:
return res
files = {}
for tuple in res["Value"]:
fileName = tuple[0]
files[fileName] = dict(zip(metadata, tuple[1:]))
return S_OK(files)
def _getDirectoryFiles(self,dirID,fileNames,metadata,allStatus=False,connection=False):
connection = self._getConnection(connection)
# metadata can be any of ['FileID','Size','UID','GID','Checksum','ChecksumType','Type','CreationDate','ModificationDate','Mode','Status']
req = "SELECT FileName,%s FROM FC_Files WHERE DirID=%d" % (intListToString(metadata),dirID)
if not allStatus:
statusIDs = []
res = self._getStatusInt('AprioriGood',connection=connection)
if res['OK']:
statusIDs.append(res['Value'])
if statusIDs:
req = "%s AND Status IN (%s)" % (req,intListToString(statusIDs))
if fileNames:
req = "%s AND FileName IN (%s)" % (req,stringListToString(fileNames))
res = self.db._query(req,connection)
if not res['OK']:
return res
files = {}
for tuple in res['Value']:
fileName = tuple[0]
files[fileName] = dict(zip(metadata,tuple[1:]))
return S_OK(files)
def __findFullPathDatasets( self, datasets, connection ):
dirDict = self._getDatasetDirectories( datasets )
failed = {}
successful = {}
result = self.db.dtree.findDirs( dirDict.keys() )
if not result['OK']:
return result
directoryIDs = result['Value']
directoryPaths = {}
for dirPath in dirDict:
if not dirPath in directoryIDs:
for dsName in dirDict[dirPath]:
dname = '%s/%s' % (dirPath,dsName)
dname = dname.replace('//','/')
failed[dname] = 'No such dataset or directory'
else:
directoryPaths[directoryIDs[dirPath]] = dirPath
wheres = []
for dirPath in directoryIDs:
dsNames = dirDict[dirPath]
dirID = directoryIDs[dirPath]
wheres.append( "( DirID=%d AND DatasetName IN (%s) )" % ( dirID, stringListToString( dsNames ) ) )
req = "SELECT DatasetName,DirID,DatasetID FROM FC_MetaDatasets WHERE %s" % " OR ".join( wheres )
result = self.db._query(req,connection)
if not result['OK']:
return result
for dsName, dirID, dsID in result['Value']:
dname = '%s/%s' % ( directoryPaths[dirID], dsName )
dname = dname.replace('//','/')
successful[dname] = { "DirID": dirID, "DatasetID": dsID }
for dataset in datasets:
if not dataset in successful:
failed[dataset] = "No such dataset"
return S_OK({"Successful":successful,"Failed":failed})
def getRequestFileStatus( self, requestID, files ):
""" collect and return all file statuses given :requestID:
:param int requestID: Requests.RequestID
:param list files: [ Files.LFN ]
"""
req = "SELECT DISTINCT `SubRequestID` FROM `SubRequests` WHERE `RequestID`=%d;" % requestID
res = self._query( req )
if not res['OK']:
return res
subRequests = []
for subRequestID in res['Value'][0]:
subRequests.append( subRequestID )
req = "SELECT `LFN`,`Status` FROM `Files` WHERE `SubRequestID` IN (%s) AND `LFN` in (%s);" %\
( intListToString( subRequests ), stringListToString( files ) )
res = self._query( req )
if not res['OK']:
return res
files = {}
for lfn, status in res['Value']:
files[lfn] = status
return S_OK( files )
def _findDirectories(self,paths,metadata=[]):
""" Find file ID if it exists for the given list of LFNs """
startTime = time.time()
successful = {}
failed = {}
req = "SELECT DirName,DirID"
if metadata:
req = "%s,%s" % (req,intListToString(metadata))
req = "%s FROM DirectoryInfo WHERE DirName IN (%s)" % (req,stringListToString(paths))
res = self.db._query(req)
if not res['OK']:
return res
for tuple in res['Value']:
dirName = tuple[0]
dirID = tuple[1]
metaDict = {'DirID':dirID}
metaDict.update(dict(zip(metadata,tuple[2:])))
successful[dirName] = metaDict
for path in paths:
if not successful.has_key(path):
failed[path] = 'No such file or directory'
return S_OK({"Successful":successful,"Failed":failed})
def getFTSJobList(self, statusList=None, limit=500):
""" select FTS jobs with statuses in :statusList: """
statusList = statusList if statusList else list(FTSJob.INITSTATES +
FTSJob.TRANSSTATES)
query = "SELECT * FROM `FTSJob` WHERE `Status` IN (%s) ORDER BY `LastUpdate` DESC LIMIT %s;" % (
stringListToString(statusList), limit)
trn = self._transaction([query])
if not trn['OK']:
self.log.error("getFTSJobList: %s" % trn['Message'])
return trn
ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in trn['Value'][query]]
for ftsJob in ftsJobs:
query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
trn = self._transaction(query)
if not trn['OK']:
self.log.error("getFTSJobList: %s" % trn['Message'])
return trn
ftsFiles = [
FTSFile(ftsFileDict) for ftsFileDict in trn['Value'][query]
]
for ftsFile in ftsFiles:
ftsJob.addFile(ftsFile)
return S_OK(ftsJobs)
def getLFNForGUID(self, guids, connection=False):
"""Returns the lfns matching given guids"""
connection = self._getConnection(connection)
if not guids:
return S_OK({})
if not isinstance(guids, (list, tuple)):
guids = [guids]
req = "SELECT f.FileID, f.FileName, fi.GUID, f.DirID FROM FC_FileInfo fi"
req += " JOIN FC_Files f on fi.FileID = f.FileID WHERE GUID IN (%s)" % stringListToString(
guids)
res = self.db._query(req, connection)
if not res["OK"]:
return res
fileIDguid = {}
dirIDFileIDs = {}
fileIDName = {}
for fileID, fileName, guid, dirID in res["Value"]:
fileIDguid[fileID] = guid
dirIDFileIDs.setdefault(dirID, []).append(fileID)
fileIDName[fileID] = fileName
res = self.db.dtree.getDirectoryPaths(list(dirIDFileIDs))
if not res["OK"]:
return res
dirIDName = res["Value"]
guidLFN = dict([(fileIDguid[fId],
os.path.join(dirIDName[dId], fileIDName[fId]))
for dId in dirIDName for fId in dirIDFileIDs[dId]])
failedGuid = set(guids) - set(guidLFN)
failed = dict.fromkeys(failedGuid,
"GUID does not exist") if failedGuid else {}
return S_OK({"Successful": guidLFN, "Failed": failed})
def __findDirs(self,paths,metadata=['DirName']):
dirs = {}
req = "SELECT DirID,%s FROM DirectoryInfo WHERE DirName IN (%s)" % (intListToString(metadata),stringListToString(paths))
res = self.db._query(req)
if not res['OK']:
return res
if not res['Value']:
return S_OK(dirs)
for tuple in res['Value']:
dirID = tuple[0]
dirs[dirID] = dict(zip(metadata,tuple[1:]))
return S_OK(dirs)
def getRequestFileStatus( self, requestID, files ):
req = "SELECT DISTINCT SubRequestID FROM SubRequests WHERE RequestID = %d;" % requestID
res = self._query( req )
if not res['OK']:
return res
subRequests = []
for subRequestID in res['Value'][0]:
subRequests.append( subRequestID )
req = "SELECT LFN,Status from Files WHERE SubRequestID IN (%s) AND LFN in (%s);" % ( intListToString( subRequests ), stringListToString( files ) )
res = self._query( req )
if not res['OK']:
return res
files = {}
for lfn, status in res['Value']:
files[lfn] = status
return S_OK( files )
def _getDirectoryFiles(self,dirID,fileNames,metadata_input,allStatus=False,connection=False):
""" Get the metadata for files in the same directory
"""
metadata = list(metadata_input)
connection = self._getConnection(connection)
# metadata can be any of ['FileID','Size','UID','GID','Status','Checksum','CheckSumType','Type','CreationDate','ModificationDate','Mode']
req = "SELECT FileName,DirID,FileID,Size,UID,GID,Status FROM FC_Files WHERE DirID=%d" % (dirID)
if not allStatus:
statusIDs = []
res = self._getStatusInt('AprioriGood',connection=connection)
if res['OK']:
statusIDs.append(res['Value'])
if statusIDs:
req = "%s AND Status IN (%s)" % (req,intListToString(statusIDs))
if fileNames:
req = "%s AND FileName IN (%s)" % (req,stringListToString(fileNames))
res = self.db._query(req,connection)
if not res['OK']:
return res
fileNameIDs = res['Value']
if not fileNameIDs:
return S_OK({})
filesDict = {}
# If we only requested the FileIDs then there is no need to do anything else
if metadata == ['FileID']:
for fileName,dirID,fileID,size,uid,gid,status in fileNameIDs:
filesDict[fileName] = {'FileID':fileID}
return S_OK(filesDict)
# Otherwise get the additionally requested metadata from the FC_FileInfo table
files = {}
userDict = {}
groupDict = {}
for fileName,dirID,fileID,size,uid,gid,status in fileNameIDs:
filesDict[fileID] = fileName
files[fileName] = {}
if 'Size' in metadata:
files[fileName]['Size'] = size
if 'DirID' in metadata:
files[fileName]['DirID'] = dirID
if 'UID' in metadata:
files[fileName]['UID'] = uid
if uid in userDict:
owner = userDict[uid]
else:
owner = 'unknown'
result = self.db.ugManager.getUserName(uid)
if result['OK']:
owner = result['Value']
userDict[uid] = owner
files[fileName]['Owner'] = owner
if 'GID' in metadata:
files[fileName]['GID'] = gid
if gid in groupDict:
group = groupDict[gid]
else:
group = 'unknown'
result = self.db.ugManager.getGroupName(gid)
if result['OK']:
group = result['Value']
groupDict[gid] = group
files[fileName]['OwnerGroup'] = group
if 'Status' in metadata:
files[fileName]['Status'] = status
for element in ['FileID','Size','DirID','UID','GID','Status']:
if element in metadata:
metadata.remove(element)
metadata.append('FileID')
metadata.reverse()
req = "SELECT %s FROM FC_FileInfo WHERE FileID IN (%s)" % (intListToString(metadata),intListToString(filesDict.keys()))
res = self.db._query(req,connection)
if not res['OK']:
return res
for tuple in res['Value']:
fileID = tuple[0]
rowDict = dict(zip(metadata,tuple))
files[filesDict[fileID]].update(rowDict)
return S_OK(files)
def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, connection=False):
""" Get the metadata for files in the same directory
"""
metadata = list(metadata_input)
connection = self._getConnection(connection)
# metadata can be any of ['FileID','Size','UID','GID','Status','Checksum','ChecksumType',
# 'Type','CreationDate','ModificationDate','Mode']
req = "SELECT FileName,DirID,FileID,Size,UID,GID,Status FROM FC_Files WHERE DirID=%d" % (dirID)
if not allStatus:
statusIDs = []
for status in self.db.visibleFileStatus:
res = self._getStatusInt(status, connection=connection)
if res["OK"]:
statusIDs.append(res["Value"])
if statusIDs:
req = "%s AND Status IN (%s)" % (req, intListToString(statusIDs))
if fileNames:
req = "%s AND FileName IN (%s)" % (req, stringListToString(fileNames))
res = self.db._query(req, connection)
if not res["OK"]:
return res
fileNameIDs = res["Value"]
if not fileNameIDs:
return S_OK({})
filesDict = {}
# If we only requested the FileIDs then there is no need to do anything else
if metadata == ["FileID"]:
for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
filesDict[fileName] = {"FileID": fileID}
return S_OK(filesDict)
# Otherwise get the additionally requested metadata from the FC_FileInfo table
files = {}
userDict = {}
groupDict = {}
for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
filesDict[fileID] = fileName
files[fileName] = {}
if "Size" in metadata:
files[fileName]["Size"] = size
if "DirID" in metadata:
files[fileName]["DirID"] = dirID
if "UID" in metadata:
files[fileName]["UID"] = uid
if uid in userDict:
owner = userDict[uid]
else:
owner = "unknown"
result = self.db.ugManager.getUserName(uid)
if result["OK"]:
owner = result["Value"]
userDict[uid] = owner
files[fileName]["Owner"] = owner
if "GID" in metadata:
files[fileName]["GID"] = gid
if gid in groupDict:
group = groupDict[gid]
else:
group = "unknown"
result = self.db.ugManager.getGroupName(gid)
if result["OK"]:
group = result["Value"]
groupDict[gid] = group
files[fileName]["OwnerGroup"] = group
if "Status" in metadata:
files[fileName]["Status"] = self._getIntStatus(status).get("Value", status)
for element in ["FileID", "Size", "DirID", "UID", "GID", "Status"]:
if element in metadata:
metadata.remove(element)
metadata.append("FileID")
metadata.reverse()
req = "SELECT %s FROM FC_FileInfo WHERE FileID IN (%s)" % (
intListToString(metadata),
intListToString(filesDict.keys()),
)
res = self.db._query(req, connection)
if not res["OK"]:
return res
for tuple_ in res["Value"]:
fileID = tuple_[0]
rowDict = dict(zip(metadata, tuple_))
files[filesDict[fileID]].update(rowDict)
return S_OK(files)
def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, connection=False):
"""For a given directory, and eventually given file, returns all the desired metadata
:param int dirID: directory ID
:param fileNames: the list of filenames, or []
:param metadata_input: list of desired metadata.
It can be anything from (FileName, DirID, FileID, Size, UID, Owner,
GID, OwnerGroup, Status, GUID, Checksum, ChecksumType, Type, CreationDate, ModificationDate, Mode)
:param bool allStatus: if False, only displays the files whose status is in db.visibleFileStatus
:returns: S_OK(files), where files is a dictionary indexed on filename, and values are dictionary of metadata
"""
connection = self._getConnection(connection)
metadata = list(metadata_input)
if "UID" in metadata:
metadata.append("Owner")
if "GID" in metadata:
metadata.append("OwnerGroup")
if "FileID" not in metadata:
metadata.append("FileID")
# Format the filenames and status to be used in a IN clause in the sotred procedure
formatedFileNames = stringListToString(fileNames)
fStatus = stringListToString(self.db.visibleFileStatus)
specificFiles = True if len(fileNames) else False
result = self.db.executeStoredProcedureWithCursor(
"ps_get_all_info_for_files_in_dir", (dirID, specificFiles, formatedFileNames, allStatus, fStatus)
)
if not result["OK"]:
return result
fieldNames = [
"FileName",
"DirID",
"FileID",
"Size",
"UID",
"Owner",
"GID",
"OwnerGroup",
"Status",
"GUID",
"Checksum",
"ChecksumType",
"Type",
"CreationDate",
"ModificationDate",
"Mode",
]
rows = result["Value"]
files = {}
for row in rows:
rowDict = dict(zip(fieldNames, row))
fileName = rowDict["FileName"]
# Returns only the required metadata
files[fileName] = dict((key, rowDict.get(key, "Unknown metadata field")) for key in metadata)
return S_OK(files)
def _getDirectoryFiles(self,
dirID,
fileNames,
metadata_input,
allStatus=False,
connection=False):
""" Get the metadata for files in the same directory
"""
metadata = list(metadata_input)
connection = self._getConnection(connection)
# metadata can be any of ['FileID','Size','UID','GID','Status','Checksum','CheckSumType',
# 'Type','CreationDate','ModificationDate','Mode']
req = "SELECT FileName,DirID,FileID,Size,UID,GID,Status FROM FC_Files WHERE DirID=%d" % (
dirID)
if not allStatus:
statusIDs = []
for status in self.db.visibleFileStatus:
res = self._getStatusInt(status, connection=connection)
if res['OK']:
statusIDs.append(res['Value'])
if statusIDs:
req = "%s AND Status IN (%s)" % (req,
intListToString(statusIDs))
if fileNames:
req = "%s AND FileName IN (%s)" % (req,
stringListToString(fileNames))
res = self.db._query(req, connection)
if not res['OK']:
return res
fileNameIDs = res['Value']
if not fileNameIDs:
return S_OK({})
filesDict = {}
# If we only requested the FileIDs then there is no need to do anything else
if metadata == ['FileID']:
for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
filesDict[fileName] = {'FileID': fileID}
return S_OK(filesDict)
# Otherwise get the additionally requested metadata from the FC_FileInfo table
files = {}
userDict = {}
groupDict = {}
for fileName, dirID, fileID, size, uid, gid, status in fileNameIDs:
filesDict[fileID] = fileName
files[fileName] = {}
if 'Size' in metadata:
files[fileName]['Size'] = size
if 'DirID' in metadata:
files[fileName]['DirID'] = dirID
if 'UID' in metadata:
files[fileName]['UID'] = uid
if uid in userDict:
owner = userDict[uid]
else:
owner = 'unknown'
result = self.db.ugManager.getUserName(uid)
if result['OK']:
owner = result['Value']
userDict[uid] = owner
files[fileName]['Owner'] = owner
if 'GID' in metadata:
files[fileName]['GID'] = gid
if gid in groupDict:
group = groupDict[gid]
else:
group = 'unknown'
result = self.db.ugManager.getGroupName(gid)
if result['OK']:
group = result['Value']
groupDict[gid] = group
files[fileName]['OwnerGroup'] = group
if 'Status' in metadata:
files[fileName]['Status'] = status
for element in ['FileID', 'Size', 'DirID', 'UID', 'GID', 'Status']:
if element in metadata:
metadata.remove(element)
metadata.append('FileID')
metadata.reverse()
req = "SELECT %s FROM FC_FileInfo WHERE FileID IN (%s)" % (
intListToString(metadata), intListToString(filesDict.keys()))
res = self.db._query(req, connection)
if not res['OK']:
return res
for tuple_ in res['Value']:
fileID = tuple_[0]
rowDict = dict(zip(metadata, tuple_))
files[filesDict[fileID]].update(rowDict)
return S_OK(files)
