class Badger:
def __init__(self, fcClient=False):
"""Internal initialization of Badger API.
"""
if not fcClient:
_fcType = "DataManagement/FileCatalog"
self.client = FileCatalogClient(_fcType)
else:
self.client = fcClient
# self.besclient = FileCatalogClient('DataManagement/DatasetFileCatalog')
def __createQuery(metaSelections):
"""
need the function then,need not import
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
TODO...
"""
pass
def getDatasetNamePrefix(self):
"""descide the prefix of a datasetName"""
prefix = ""
result = getProxyInfo(False, False)
if result["OK"]:
userGroup = result["Value"]["group"]
if userGroup == "bes_user":
prefix = "User_"
elif userGroup == "production":
prefix = "Proc_"
return prefix
else:
return prefix
def getFilenamesByLocaldir(self, localDir):
""" get all files under the given dir
example:getFilenamesByLocaldir("/bes3fs/offline/data/663-1/4260/dst/121215/")
result = [/bes3fs/offline/data/663-1/4260/dst/121215/filename1,
/bes3fs/offline/data/663-1/4260/dst/121215/filename2,
...
]
"""
fileList = []
for rootdir, subdirs, files in os.walk(localDir):
for name in files:
fullPath = os.path.join(rootdir, name)
fileList.append(fullPath)
fileList.sort()
return fileList
def __getFileAttributes(self, fullPath):
""" get all attributes of the given file,return a attribute dict.
"""
if os.path.exists(fullPath):
type = judgeType(fullPath)
if type == "all":
obj = DataAll(fullPath)
elif type == "others":
obj = Others(fullPath)
elif type == None:
errorMes = "name if %s is not correct" % fullPath
print "cannot get attributes of %s" % fullPath
attributes = {}
return attributes
# raise TypeError(errorMes)
attributes = obj.getAttributes()
# attributes['date'] = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())
else:
attributes = {}
return attributes
def testFunction(self):
result = self.__getFileAttributes(
"/besfs2/offline/data/664-1/jpsi/dst/090613/run_0009952_All_file019_SFO-2.dst"
)
print result
def __registerDir(self, dir):
"""Internal function to register a new directory in DFC .
Returns True for success, False for failure.
"""
fc = self.client
result = fc.createDirectory(dir)
if result["OK"]:
if result["Value"]["Successful"]:
if result["Value"]["Successful"].has_key(dir):
return S_OK()
elif result["Value"]["Failed"]:
if result["Value"]["Failed"].has_key(dir):
print "Failed to create directory %s:%s" % (dir, result["Value"]["Failed"][dir])
return S_ERROR(result)
else:
return S_ERROR(result)
else:
print "Failed to create directory %s:%s" % (dir, result["Message"])
return S_ERROR(result)
def __registerFileMetadata(self, lfn, attributes):
"""Internal function to set metadata values on a given lfn.
Returns True for success, False for failure.
"""
metadataDict = {}
metadataDict["runL"] = attributes["runL"]
metadataDict["runH"] = attributes["runH"]
metadataDict["status"] = attributes["status"]
metadataDict["eventNumber"] = attributes["eventNumber"]
metadataDict["count"] = attributes["count"]
result = self.client.setMetadata(lfn, metadataDict)
if not result["OK"]:
return S_ERROR()
else:
return S_OK()
def __registerDirMetadata(self, dir, metaDict):
"""Internal function to set metadata to a directory
Returns True for success, False for failure.
"""
fc = self.client
result = fc.setMetadata(dir, metaDict)
if result["OK"]:
return S_OK()
else:
message = "Error for setting metadata %s to %s: %s" % (metaDict, dir, result["Message"])
return S_ERROR("Message")
def __dirExists(self, dir, parentDir):
""" Internal function to check whether 'dir' is the subdirectory of 'parentDir'
Returns 1 for Yes, 0 for NO
"""
fc = self.client
dir_exists = 0
result = fc.listDirectory(parentDir)
if result["OK"]:
for i, v in enumerate(result["Value"]["Successful"][parentDir]["SubDirs"]):
if v == dir:
dir_exists = 1
break
else:
print "Failed to list subdirectories of %s:%s" % (parentDir, result["Message"])
return dir_exists
def __registerSubDirs(self, dirs_dict, dirs_meta):
"""Internal function to create directories in dirs_dict
Returns True for sucess, False for failure
"""
creation_ok = True
for dir in dirs_dict:
if (dir != "dir_file") & (dir != "dir_data_mc"):
if self.__registerDir(dirs_meta[dir][0])["OK"]:
result = self.__registerDirMetadata(dirs_meta[dir][0], {dir.split("_")[1]: dirs_meta[dir][1]})
if not result["OK"]:
creation_ok = False
break
else:
print "Failed to create %s" % dir
creation_ok = False
break
else:
if not self.__registerDir(dirs_meta[dir])["OK"]:
print "Failed to create %s" % dir
creation_ok = False
break
return creation_ok
def registerHierarchicalDir(self, metaDict, rootDir="/bes"):
"""
Create a hierarchical directory according the metadata dictionary
Return created directory for sucess,if this directory has been created, return this existing directory .
Structure of the hierarchical directory:
for real data:/bes/File/resonance/boss version/data/eventType/round
for mc data:/bes/File/resonance/boss version/mc/eventType/round/streamId
The eventType of all real datas is all.
Example:
>>>metaDict = {'dataType': 'dst', 'eventType': 'all', 'streamId': 'stream0','resonance': 'psipp', 'round':'round01','bossVer': '6.6.1'}
>>>badger.registerHierarchicalDir(metaDic)
1
"""
# Save about 20 lines compared with last one
fc = self.client
dir_exists = 0
# 0 for failure,1 for success,2 for existing directory
creation_OK = 0
lastDirMetaDict = {"dataType": metaDict["dataType"], "streamId": metaDict["streamId"]}
dir_file = rootDir + "/File"
dir_resonance = dir_file + "/" + metaDict["resonance"]
dir_bossVer = dir_resonance + "/" + metaDict["bossVer"]
if metaDict["streamId"] == "stream0":
dir_data_mc = dir_bossVer + "/data"
else:
dir_data_mc = dir_bossVer + "/mc"
dir_eventType = dir_data_mc + "/" + metaDict["eventType"]
dir_round = dir_eventType + "/" + metaDict["round"]
dir_streamId = dir_round + "/" + metaDict["streamId"]
# if dir_round has been created,create_round=1
create_round = 0
dirs_dict = ["dir_file", "dir_resonance", "dir_bossVer", "dir_data_mc", "dir_eventType", "dir_round"]
dirs_meta = {
"dir_file": dir_file,
"dir_data_mc": dir_data_mc,
"dir_resonance": [dir_resonance, metaDict["resonance"]],
"dir_bossVer": [dir_bossVer, metaDict["bossVer"]],
"dir_eventType": [dir_eventType, metaDict["eventType"]],
"dir_round": [dir_round, metaDict["round"]],
}
dir_exists = self.__dirExists(dir_file, rootDir)
if not dir_exists:
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_resonance, dir_file)
if not dir_exists:
dirs_dict = dirs_dict[1:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_bossVer, dir_resonance)
if not dir_exists:
dirs_dict = dirs_dict[2:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_data_mc, dir_bossVer)
if not dir_exists:
dirs_dict = dirs_dict[3:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_eventType, dir_data_mc)
if not dir_exists:
dirs_dict = dirs_dict[4:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_round, dir_eventType)
if not dir_exists:
dirs_dict = dirs_dict[5:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
create_round = 1
if create_round:
if metaDict["streamId"] != "stream0":
dir_exists = self.__dirExists(dir_streamId, dir_round)
if not dir_exists:
if self.__registerDir(dir_streamId)["OK"]:
result = self.__registerDirMetadata(dir_streamId, {"streamId": metaDict["streamId"]})
if result["OK"]:
result = self.__registerDirMetadata(dir_streamId, lastDirMetaDict)
if result["OK"]:
creation_OK = 1
else:
creation_OK = 2
else:
result = self.__registerDirMetadata(dir_round, lastDirMetaDict)
if result["OK"]:
creation_OK = 1
if (creation_OK == 1) | (creation_OK == 2):
if metaDict["streamId"] == "stream0":
return dir_round
else:
return dir_streamId
##########################################################################################
# dir options
def removeDir(self, dir):
"""remove the dir include files and subdirs
"""
result = self.client.listDirectory(dir)
if result["OK"]:
if not result["Value"]["Successful"][dir]["Files"] and not result["Value"]["Successful"][dir]["SubDirs"]:
# print 'no file and subDirs in this dir'
self.client.removeDirectory(dir)
return S_OK()
else:
if result["Value"]["Successful"][dir]["Files"]:
for file in result["Value"]["Successful"][dir]["Files"]:
self.client.removeFile(file)
else:
for subdir in result["Value"]["Successful"][dir]["SubDirs"]:
self.removeDir(subdir)
self.removeDir(dir)
def listDir(self, dir):
"""list the files under the given DFC dir"""
fileList = []
result = self.client.listDirectory(dir)
if result["OK"]:
if result["Value"]["Successful"][dir]["Files"]:
fileList = result["Value"]["Successful"][dir]["Files"].keys()
fileList.sort()
else:
print "no files under this dir"
return fileList
def getDirMetaVal(self, dir):
"""list the registed metadata value of the given dir"""
result = self.client.getDirectoryMetadata(dir)
if result["OK"]:
return result["Value"]
else:
print "Failed to get meta Value of the directory"
return {}
#################################################################################
# meta fields operations
#
def addNewFields(self, fieldName, fieldType, metaType="-d"):
"""add new fields,if metaType is '-f',add file field,
fileType is datatpye in MySQL notation
"""
result = self.client.addMetadataField(fieldName, fieldType, metaType)
if not result["OK"]:
return S_ERROR(result)
else:
return S_OK()
def deleteMetaField(self, fieldName):
"""delete a exist metafield"""
result = self.client.deleteMetadataField(fieldName)
if not result["OK"]:
return S_ERROR(result)
else:
return S_OK()
def getAllFields(self):
"""get all meta fields,include file metafield and dir metafield.
"""
result = self.client.getMetadataFields()
if not result["OK"]:
return S_ERROR(result["Message"])
else:
return result["Value"]
def registerFileMetadata(self, lfn, metaDict):
"""Add file level metadata to an entry
True for success, False for failure
(maybe used to registerNewMetadata
Example:
>>>lfn = '/bes/File/psipp/6.6.1/data/all/exp1/run_0011414_All_file001_SFO-1'
>>>entryDict = {'runL':1000,'runH':898898}
>>>badger.registerFileMetadata(lfn,entryDict)
True
"""
fc = self.client
result = fc.setMetadata(lfn, metaDict)
if result["OK"]:
return S_OK()
else:
print "Error:%s" % (result["Message"])
return S_ERROR(result["Message"])
#####################################################################
# File Options
def registerFile(self, lfn, dfcAttrDict):
"""Register a new file in the DFC.
"""
# TODO:need more tests,if directory of file doesn't exist,
# addFile will create it without setting any metadata(lin lei)
# need to check whether directory of file exists in dfc?(lin lei)
# pass
fc = self.client
result = fc.addFile({lfn: dfcAttrDict})
if result["OK"]:
if result["Value"]["Successful"]:
if result["Value"]["Successful"].has_key(lfn):
return S_OK()
elif result["Value"]["Failed"]:
if result["Value"]["Failed"].has_key(lfn):
print "Failed to add this file:", result["Value"]["Failed"][lfn]
return S_ERROR()
else:
print "Failed to add this file :", result["Message"]
return S_ERROR()
# need to register file (inc. creating appropriate directory
# if it doesn't already exist; and register metadata for that
# file / directory
# Q: how / where to pass the metadata?
def getFileMetaVal(self, lfn):
"""get the File Meta Value of given file
"""
result = self.client.getFileUserMetadata(lfn)
if result["OK"]:
return result["Value"]
else:
print "Failed to get meta Value of this file"
return {}
def reCalcCount(self, fileList, plus=True):
"""calculate the value of metadata 'count',when a file contain in a dataset
count+1,when del a dataset,then all file in this dataset count -1
default plus=True,means count+1,if count-1,set plus=False
return the value of count, count = -1 means error.
NOTE:this function should only be called when create or delete a dataset.
"""
countDict = {}
if type(fileList) != type([]):
fileList = [fileList]
for file in fileList:
result = self.getFileMetaVal(file)
if len(result) != 0:
count = result["count"]
if plus:
count += 1
else:
if count > 0:
count -= 1
cDict = {"count": count}
self.registerFileMetadata(file, cDict)
countDict[file] = count
else:
print "Failed reCalculate value of count of file %s" % file
countDict[file] = -1
return countDict
def removeFile(self, lfn):
"""remove file on DFC
"""
result = self.client.removeFile(lfn)
if not result["OK"]:
return S_ERROR(result)
else:
return S_OK()
def getPFN(self, lfn):
"""get replicas by lfn"""
result = self.client.getReplicas(lfn)
# print result
if not result["OK"]:
return S_ERROR(result)
else:
return S_OK(result["Value"]["Successful"][lfn]["IHEPD-USER"])
def getSize(self, lfns):
"""get the size of the given lfn"""
result = self.client.getFileSize(lfns)
if result["OK"]:
if result["Value"]["Successful"]:
retVal = result["Value"]["Successful"]
else:
retVal = {}
return retVal
def getFilesByMetadataQuery(self, query):
"""Return a list of LFNs satisfying given query conditions.
Example usage:
>>> brunH_GT_29756adger.getFilesByMetadataQuery('resonance=jpsi bossVer=6.5.5 round=exp1')
['/bes/File/jpsi/6.5.5/data/all/exp1/file1', .....]
"""
# TODO: checking of output, error catching
fc = self.client
# TODO: calling the FileCatalog CLI object and its private method
# is not a good way of doing this! but use it to allow construction of
# the query meantime, until createQuery is made a public method
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(query)
result = fc.findFilesByMetadata(metadataDict, "/")
if result["OK"]:
lfns = fc.findFilesByMetadata(metadataDict, "/")["Value"]
lfns.sort()
return lfns
else:
print "ERROR: No files found which match query conditions."
return None
def uploadAndRegisterFiles(self, fileList, SE="IHEPD-USER", guid=None, ePoint=""):
"""upload a set of files to SE and register it in DFC.
user input the directory of localfile.
argument:
ePoint is the energy point,for scan data
we can treat localDir as a kind of datasetName.
"""
result_OK = 1
errorList = []
# fileList = self.getFilenamesByLocaldir(localDir)
for fullpath in fileList:
# get the attributes of the file
fileAttr = self.__getFileAttributes(fullpath)
if len(fileAttr) == 0:
print "failed to get file %s attributes" % fullpath
return S_ERROR("failed to get file attributes")
# create dir and set dirMetadata to associated dir
lastDir = self.registerHierarchicalDir(fileAttr, rootDir="/bes")
dirMeta = self.getDirMetaVal(lastDir)
if not (dirMeta.has_key("jobOptions") or dirMeta.has_key("description")):
lastDirMetaDict = {}
lastDirMetaDict["jobOptions"] = fileAttr["jobOptions"]
lastDirMetaDict["description"] = fileAttr["description"]
try:
self.__registerDirMetadata(lastDir, lastDirMetaDict)
except:
pass
if len(ePoint):
lastDir = lastDir + os.sep + ePoint
lfn = lastDir + os.sep + fileAttr["LFN"]
# upload and register file.
dirac = Dirac()
result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True)
# register file metadata
if not result["OK"]:
print "ERROR %s" % (result["Message"])
# return S_ERROR(result['Message'])
errorList.append(fullpath)
result_OK = 0
else:
result = self.__registerFileMetadata(lfn, fileAttr)
if not result["OK"]:
result_OK = 0
print "failed to register file metadata"
if result_OK:
return S_OK()
else:
return S_ERROR(errorList)
def downloadFilesByFilelist(self, fileList, destDir=""):
"""downLoad a set of files form SE.
use getFilesByFilelist() get a list of lfns and download these files.
fileList get from function getFilesByDatesetName()
Example usage:
>>>badger.downloadFilesByFilelist(fileList)
"""
errorDict = {}
dirac = Dirac()
# fileList = self.getFilesByDatasetName(dataset_name)
for lfn in fileList:
result = dirac.getFile(lfn, destDir, printOutput=False)
if not result["OK"]:
errorDict[lfn] = result["Message"]
if errorDict:
serr = S_ERROR()
serr["errorDict"] = errorDict
return serr
else:
return S_OK("File download successfully.")
####################################################################
# dataset functions
#
def registerDataset(self, datasetName, path, conditions):
"""Register a new dataset in DFC. Takes dataset name and string with
conditions for new dataset as arguments.
datasetname format:
resonance_BossVer_eventtype_round_runL_runH_stream0_datatype
resonance_BossVer_eventtype_round_runL_runH_streamID_datatype
type(conditions) is str,like "resonance=jpsi bossVer=655 round=round1"
"""
fc = self.client
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(conditions)
metadataDict["Path"] = path
result = fc.addDataset(datasetName, metadataDict)
if not result["OK"]:
print ("Error: %s" % result["Message"])
return S_ERROR()
else:
print "Added dataset %s with conditions %s" % (datasetName, conditions)
return S_OK()
def getDatasetDescription(self, datasetName):
"""Return a string containing a description of metadata with which
the given dataset was defined.
Example usage:
>>> result = badger.getDatasetDescription('psipp_661_data_all_exp2')
"""
result = self.client.getDatasetParameters(datasetName)
if not result["OK"]:
print "ERROR: failed to get status of dataset:", result["Message"]
else:
parDict = result["Value"]
for par, value in parDict.items():
print par.rjust(20), ":", value
def getDatasetMetadata(self, datasetName):
"""Return a dict containing a description of metadata with which
the given dataset was defined.
Example usage:
>>> result = badger.getDatasetMetadata('psipp_661_data_all_exp2')
"""
result = self.client.getDatasetParameters(datasetName)
if not result["OK"]:
print "ERROR: failed to get status of dataset:", result["Message"]
return S_ERROR(result["Message"])
else:
parDict = result["Value"]
return S_OK(parDict)
def removeDataset(self, datasetName):
"""remove a dataset
"""
result = self.client.removeDataset(datasetName)
if not result["OK"]:
print "ERROR: failed to remove dataset:", result["Message"]
else:
print "Successfully removed dataset", datasetName
def checkDataset(self, datasetName):
""" check if the dataset parameters are still valid
"""
result = self.client.checkDataset(datasetName)
if not result["OK"]:
print "ERROR: failed to check dataset:", result["Message"]
else:
changeDict = result["Value"]
if not changeDict:
print "Dataset is not changed"
else:
print "Dataset changed:"
for par in changeDict:
print " ", par, ": ", changeDict[par][0], "->", changeDict[par][1]
def updateDataset(self, datasetName):
""" Update the given dataset parameters
"""
result = self.client.updateDataset(datasetName)
if not result["OK"]:
print "ERROR: failed to update dataset:", result["Message"]
else:
print "Successfully updated dataset", datasetName
def freezeDataset(self, datasetName):
""" Freeze the given dataset
"""
result = self.client.freezeDataset(datasetName)
if not result["OK"]:
print "ERROR: failed to freeze dataset:", result["Message"]
else:
print "Successfully frozen dataset", datasetName
def releaseDataset(self, datasetName):
""" Release the given dataset
"""
result = self.client.releaseDataset(datasetName)
if not result["OK"]:
print "ERROR: failed to release dataset:", result["Message"]
else:
print "Successfully released dataset", datasetName
def getFilesByDatasetName(self, datasetName):
"""Return a list of LFNs in the given dataset.
Example usage:
>>> badger.getFilesByDatasetName('psipp_661_data_all_exp2')
['/bes/File/psipp/6.6.1/data/all/exp2/file1', .....]
"""
fc = self.client
result = returnSingleResult(fc.getDatasetFiles(datasetName))
if result["OK"]:
lfns = result["Value"]
lfns.sort()
return S_OK(lfns)
else:
print "ERROR: Dataset", datasetName, " not found"
return S_ERROR(result)
def listDatasets(self):
"""list the exist dataset"""
datasetName = ""
result = self.client.getDatasets(datasetName)
if not result["OK"]:
print "ERROR:failed to get datasets"
return
datasetDict = result["Value"]
for dName in datasetDict.keys():
print dName
def checkDatasetIntegrity():
pass
class Badger:
def __init__(self, fcClient=False):
"""Internal initialization of Badger API.
"""
if not fcClient:
_fcType = 'DataManagement/FileCatalog'
self.client = FileCatalogClient(_fcType)
else:
self.client = fcClient
self.besclient = FileCatalogClient('DataManagement/DatasetFileCatalog')
def __getFilenamesByLocaldir(self, localDir):
""" get all files under the given dir
example:__getFilenamesByLocaldir("/bes3fs/offline/data/663-1/4260/dst/121215/")
result = [/bes3fs/offline/data/663-1/4260/dst/121215/filename1,
/bes3fs/offline/data/663-1/4260/dst/121215/filename2,
...
]
"""
fileList = []
for rootdir, subdirs, files in os.walk(localDir):
for name in files:
fullPath = os.path.join(rootdir, name)
fileList.append(fullPath)
return fileList
def __getFileAttributes(self, fullPath):
""" get all attributes of the given file,return a attribute dict.
"""
if os.path.exists(fullPath):
type = judgeType(fullPath)
if type == "all":
obj = DataAll(fullPath)
elif type == "others":
obj = Others(fullPath)
elif type == None:
errorMes = "name if %s is not correct" % fullPath
print "cannot get attributes of %s" % fullPath
return S_ERROR(errorMes)
attributes = obj.getAttributes()
attributes['date'] = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime())
return attributes
def testFunction(self):
result = self.__getFilenamesByLocaldir(
'/besfs2/offline/data/664-1/jpsi/dst')
for item in result:
print item
print len(result)
def __registerDir(self, dir):
"""Internal function to register a new directory in DFC .
Returns True for success, False for failure.
"""
fc = self.client
result = fc.createDirectory(dir)
if result['OK']:
if result['Value']['Successful']:
if result['Value']['Successful'].has_key(dir):
return S_OK()
elif result['Value']['Failed']:
if result['Value']['Failed'].has_key(dir):
print 'Failed to create directory %s:%s' % (
dir, result['Value']['Failed'][dir])
return S_ERROR()
else:
print 'Failed to create directory %s:%s' % (dir, result['Message'])
return S_ERROR()
def __registerFileMetadata(self, lfn, attributes):
"""Internal function to set metadata values on a given lfn.
Returns True for success, False for failure.
"""
metadataDict = {}
metadataDict['dataType'] = attributes['dataType']
metadataDict['runL'] = attributes['runL']
metadataDict['runH'] = attributes['runH']
metadataDict['status'] = attributes['status']
metadataDict['description'] = attributes['description']
metadataDict['date'] = attributes['date']
metadataDict['LFN'] = attributes['LFN']
metadataDict['PFN'] = attributes['PFN']
metadataDict['eventNum'] = attributes['eventNum']
metadataDict['fileSize'] = attributes['fileSize']
result = self.client.setMetadata(lfn, metadataDict)
if not result['OK']:
return S_ERROR()
else:
return S_OK()
def __registerDirMetadata(self, dir, metaDict):
"""Internal function to set metadata to a directory
Returns True for success, False for failure.
"""
fc = self.client
result = fc.setMetadata(dir, metaDict)
if result['OK']:
return S_OK()
else:
print("Error for setting metadata %s to %s: %s" %
(metaDict, dir, result['Message']))
return S_ERROR(result['Message'])
def __dirExists(self, dir, parentDir):
""" Internal function to check whether 'dir' is the subdirectory of 'parentDir'
Returns 1 for Yes, 0 for NO
"""
fc = self.client
dir_exists = 0
result = fc.listDirectory(parentDir)
if result['OK']:
for i, v in enumerate(
result['Value']['Successful'][parentDir]['SubDirs']):
if v == dir:
dir_exists = 1
break
else:
print 'Failed to list subdirectories of %s:%s' % (
parentDir, result['Message'])
return dir_exists
def __registerSubDirs(self, dirs_dict, dirs_meta):
"""Internal function to create directories in dirs_dict
Returns True for sucess, False for failure
"""
creation_ok = True
for dir in dirs_dict:
if (dir != 'dir_file') & (dir != 'dir_data_mc'):
if self.__registerDir(dirs_meta[dir][0])['OK']:
result = self.__registerDirMetadata(
dirs_meta[dir][0],
{dir.split('_')[1]: dirs_meta[dir][1]})
if not result['OK']:
creation_ok = False
break
else:
print 'Failed to create %s' % dir
creation_ok = False
break
else:
if not self.__registerDir(dirs_meta[dir])['OK']:
print 'Failed to create %s' % dir
creation_ok = False
break
return creation_ok
def registerHierarchicalDir(self, metaDict, rootDir='/bes'):
"""
Create a hierarchical directory according the metadata dictionary
Return created directory for sucess,if this directory has been created, return this existing directory .
Structure of the hierarchical directory:
for real data:/bes/File/resonance/boss version/data/eventType/round
for mc data:/bes/File/resonance/boss version/mc/eventType/round/streamId
The eventType of all real datas is all.
Example:
>>>metaDict = {'dataType': 'dst', 'eventType': 'all', 'streamId': 'stream0','resonance': 'psipp', 'round':'round01','bossVer': '6.6.1'}
>>>badger.registerHierarchicalDir(metaDic)
1
"""
#Save about 20 lines compared with last one
fc = self.client
dir_exists = 0
#0 for failure,1 for success,2 for existing directory
creation_OK = 0
lastDirMetaDict = {
'dataType': metaDict['dataType'],
'streamId': metaDict['streamId']
}
dir_file = rootDir + '/File'
dir_resonance = dir_file + '/' + metaDict['resonance']
dir_bossVer = dir_resonance + '/' + metaDict['bossVer']
if metaDict['streamId'] == 'stream0':
dir_data_mc = dir_bossVer + '/data'
else:
dir_data_mc = dir_bossVer + '/mc'
dir_eventType = dir_data_mc + '/' + metaDict['eventType']
dir_round = dir_eventType + '/' + metaDict['round']
dir_streamId = dir_round + '/' + metaDict['streamId']
# if dir_round has been created,create_round=1
create_round = 0
dirs_dict = [
'dir_file', 'dir_resonance', 'dir_bossVer', 'dir_data_mc',
'dir_eventType', 'dir_round'
]
dirs_meta = {
'dir_file': dir_file,
'dir_data_mc': dir_data_mc,
'dir_resonance': [dir_resonance, metaDict['resonance']],
'dir_bossVer': [dir_bossVer, metaDict['bossVer']],
'dir_eventType': [dir_eventType, metaDict['eventType']],
'dir_round': [dir_round, metaDict['round']]
}
dir_exists = self.__dirExists(dir_file, rootDir)
if not dir_exists:
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_resonance, dir_file)
if not dir_exists:
dirs_dict = dirs_dict[1:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_bossVer, dir_resonance)
if not dir_exists:
dirs_dict = dirs_dict[2:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_data_mc, dir_bossVer)
if not dir_exists:
dirs_dict = dirs_dict[3:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_eventType,
dir_data_mc)
if not dir_exists:
dirs_dict = dirs_dict[4:]
result = self.__registerSubDirs(
dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(
dir_round, dir_eventType)
if not dir_exists:
dirs_dict = dirs_dict[5:]
result = self.__registerSubDirs(
dirs_dict, dirs_meta)
if result:
create_round = 1
else:
create_round = 1
if create_round:
if metaDict['streamId'] != "stream0":
dir_exists = self.__dirExists(dir_streamId, dir_round)
if not dir_exists:
if self.__registerDir(dir_streamId)['OK']:
result = self.__registerDirMetadata(
dir_streamId, {'streamId': metaDict['streamId']})
if result['OK']:
result = self.__registerDirMetadata(
dir_streamId, lastDirMetaDict)
if result['OK']:
creation_OK = 1
else:
creation_OK = 2
else:
result = self.__registerDirMetadata(dir_round, lastDirMetaDict)
if result['OK']:
creation_OK = 1
if (creation_OK == 1) | (creation_OK == 2):
if metaDict['streamId'] == "stream0":
return dir_round
else:
return dir_streamId
def removeDir(self, dir):
"""remove the dir include files and subdirs
"""
result = self.client.listDirectory(dir)
if result['OK']:
if not result['Value']['Successful'][dir]['Files'] and not result[
'Value']['Successful'][dir]['SubDirs']:
print 'no file and subDirs in this dir'
self.client.removeDirectory(dir)
return S_OK()
else:
if result['Value']['Successful'][dir]['Files']:
for file in result['Value']['Successful'][dir]['Files']:
self.client.removeFile(file)
else:
for subdir in result['Value']['Successful'][dir][
'SubDirs']:
self.removeDir(subdir)
self.removeDir(dir)
def registerFileMetadata(self, lfn, metaDict):
"""Add file level metadata to an entry
True for success, False for failure
(maybe used to registerNewMetadata
Example:
>>>lfn = '/bes/File/psipp/6.6.1/data/all/exp1/run_0011414_All_file001_SFO-1'
>>>entryDict = {'runL':1000,'runH':898898}
>>>badger.registerFileMetadata(lfn,entryDict)
True
"""
fc = self.client
result = fc.setMetadata(lfn, metaDict)
if result['OK']:
return S_OK()
else:
print 'Error:%s' % (result['Message'])
return S_ERROR(result['Message'])
#################################################################################
# meta fields operations
#
def addNewFields(self, fieldName, fieldType, metaType='-d'):
"""add new fields,if metaType is '-f',add file field,
fileType is datatpye in MySQL notation
"""
result = self.client.addMetadataField(fieldName, fieldType, metaType)
if not result['OK']:
return S_ERROR(result)
else:
return S_OK()
def deleteMetaField(self, fieldName):
"""delete a exist metafield"""
result = self.client.deleteMetadataField(fieldName)
if not result['OK']:
return S_ERROR(result)
else:
return S_OK()
def getAllFields(self):
"""get all meta fields,include file metafield and dir metafield.
"""
result = self.client.getMetadataFields()
if not result['OK']:
return S_ERROR(result['Message'])
else:
return result['Value']
#####################################################################
def registerFile(self, lfn, dfcAttrDict):
"""Register a new file in the DFC.
"""
#TODO:need more tests,if directory of file doesn't exist,
#addFile will create it without setting any metadata(lin lei)
#need to check whether directory of file exists in dfc?(lin lei)
#pass
fc = self.client
result = fc.addFile({lfn: dfcAttrDict})
if result['OK']:
if result['Value']['Successful']:
if result['Value']['Successful'].has_key(lfn):
return S_OK()
elif result['Value']['Failed']:
if result['Value']['Failed'].has_key(lfn):
print 'Failed to add this file:', result['Value'][
'Failed'][lfn]
return S_ERROR()
else:
print 'Failed to add this file :', result['Message']
return S_ERROR()
# need to register file (inc. creating appropriate directory
# if it doesn't already exist; and register metadata for that
# file / directory
# Q: how / where to pass the metadata?
def removeFile(self, lfn):
"""remove file on DFC
"""
result = self.client.removeFile(lfn)
if not result['OK']:
return S_ERROR(result)
else:
return S_OK()
def uploadAndRegisterFiles(self, localDir, SE='IHEP-USER', guid=None):
"""upload a set of files to SE and register it in DFC.
user input the directory of localfile.
we can treat localDir as a kind of datasetName.
"""
result_OK = 1
errorList = []
fileList = self.__getFilenamesByLocaldir(localDir)
for fullpath in fileList[:50]:
#get the attributes of the file
print fullpath
fileAttr = self.__getFileAttributes(fullpath)
#create dir and set dirMetadata to associated dir
metaDict = {}
metaDict['dataType'] = fileAttr['dataType']
metaDict['eventType'] = fileAttr['eventType']
metaDict['streamId'] = fileAttr['streamId']
metaDict['resonance'] = fileAttr['resonance']
metaDict['round'] = fileAttr['round']
metaDict['bossVer'] = fileAttr['bossVer']
lastDir = self.registerHierarchicalDir(metaDict,
rootDir='/zhanggang_test')
lfn = lastDir + os.sep + fileAttr['LFN']
fileAttr['LFN'] = lfn
#upload and register file.
dirac = Dirac()
result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True)
#register file metadata
if not result['OK']:
print 'ERROR %s' % (result['Message'])
#return S_ERROR(result['Message'])
errorList.append(fullpath)
result_OK = 0
else:
#get the truely PFN
storageElement = StorageElement(SE)
res = storageElement.getPfnForLfn(lfn)
destPfn = res['Value']
fileAttr['PFN'] = destPfn
result = self.__registerFileMetadata(lfn, fileAttr)
if not result['OK']:
result_OK = 0
print "failed to register file metadata"
if result_OK:
return S_OK()
else:
return S_ERROR(errorList)
####################################################################
# dataset functions
#
def registerDataset(self, dataset_name, conditions):
"""Register a new dataset in DFC. Takes dataset name and string with
conditions for new dataset as arguments.
datasetname format:
"resonance_BossVer_eventtype_round_runL_runH_stream0_datatype
example:psip_655_all_round01_8093_9025_stream0_dst
resonance_BossVer_eventtype_round_runL_runH_streamID_datatype
example:psip_655_inc_round01_8093_9025_stream1_dst
example:psipp_655_user1_round01_11414_13988_stream1_dst"
"""
pass
# need to think about how datasets are defined
# format for passing the dataset conditions?
fc = self.client
setDict = {}
for cond in conditions:
key, value = cond.split('=')
setDict[key] = value
result = fc.addMetadataSet(dataset_name, setDict)
if not result['OK']:
print("Error: %s" % result['Message'])
else:
print "Added dataset %s with conditions %s" % (dataset_name,
conditions)
def getFilesByDatasetName(self, dataset_name):
"""Return a list of LFNs in the given dataset.
Example usage:
>>> badger.getFilesByDatasetName('psipp_661_data_all_exp2')
['/bes/File/psipp/6.6.1/data/all/exp2/file1', .....]
"""
fc = self.client
#sfc = self.besclient
result = fc.getMetadataSet(dataset_name, True)
if result['Value']:
metadataDict = result['Value']
result = fc.findFilesByMetadata(metadataDict, '/')
lfns = result['Value']
lfns.sort()
dirs = fc.findDirectoriesByMetadata(metadataDict)
return lfns
else:
print "ERROR: Dataset", dataset_name, " not found"
return S_ERROR(result)
def getFilesByMetadataQuery(self, query):
"""Return a list of LFNs satisfying given query conditions.
Example usage:
>>> badger.getFilesByMetadataQuery('resonance=jpsi bossVer=6.5.5 round=exp1')
['/bes/File/jpsi/6.5.5/data/all/exp1/file1', .....]
"""
#TODO: checking of output, error catching
fc = self.client
#TODO: calling the FileCatalog CLI object and its private method
# is not a good way of doing this! but use it to allow construction of
# the query meantime, until createQuery is made a public method
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(query)
result = fc.findFilesByMetadata(metadataDict, '/')
if result['OK']:
lfns = fc.findFilesByMetadata(metadataDict, '/')['Value']
lfns.sort()
return lfns
else:
print "ERROR: No files found which match query conditions."
return None
def getDatasetDescription(self, dataset_name):
"""Return a string containing a description of metadata with which
the given dataset was defined.
Example usage:
>>> result = badger.getDatasetDescription('psipp_661_data_all_exp2')
>>> print result
Dataset psipp_661_data_all_exp2 was defined with the following metadata conditions:
round : exp2
bossVer : 6.6.1
resonance : psipp
"""
#TODO: keep this as separate method, or just return description with LFNs?
fc = self.client
result = fc.getMetadataSet(dataset_name, True)
if result['Value']:
metadataDict = result['Value']
# give user a reminder of what this dataset's definition is
dataset_desc = ''
dataset_desc += \
'Dataset %s was defined with the following metadata conditions: \n ' \
% dataset_name
for key in metadataDict:
dataset_desc += '%s : %s \n' % (key, metadataDict[key])
else:
dataset_desc = 'Error: dataset %s is not defined.' % dataset_name
return dataset_desc
def listDatasets(self):
"""list the exist dataset"""
result = self.besclient.listMetadataSets()
if not result['OK']:
return S_ERROR(result)
else:
return result['Value']
def downloadFilesByDatasetName(self, dataset_name):
"""downLoad a set of files form SE.
use getFilesByDatasetName() get a list of lfns and download these files.
Example usage:
>>>badger.downloadFilesByDatasetName('psipp_661_data_all_exp2')i
"""
dirac = Dirac()
fileList = self.getFilesByDatasetName(dataset_name)
result = dirac.getFile(fileList, printOutput=True)
if not result['OK']:
print 'ERROR %s' % (result['Message'])
return S_ERROR(result['Message'])
else:
return S_OK()
def checkDatasetIntegrity():
pass
class Badger:
def __init__(self, fcClient=False):
"""Internal initialization of Badger API.
"""
if not fcClient:
_fcType = 'DataManagement/FileCatalog'
self.client = FileCatalogClient(_fcType)
else:
self.client = fcClient
#self.besclient = FileCatalogClient('DataManagement/DatasetFileCatalog')
def __createQuery(metaSelections):
"""
need the function then,need not import
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
TODO...
"""
pass
def getDatasetNamePrefix(self):
"""descide the prefix of a datasetName"""
prefix = ''
result = getProxyInfo(False, False)
if result['OK']:
userGroup = result['Value']['group']
if userGroup == 'bes_user':
prefix = 'User_'
elif userGroup == 'production':
prefix = 'Proc_'
return prefix
else:
return prefix
def getFilenamesByLocaldir(self, localDir):
""" get all files under the given dir
example:getFilenamesByLocaldir("/bes3fs/offline/data/663-1/4260/dst/121215/")
result = [/bes3fs/offline/data/663-1/4260/dst/121215/filename1,
/bes3fs/offline/data/663-1/4260/dst/121215/filename2,
...
]
"""
fileList = []
for rootdir, subdirs, files in os.walk(localDir):
for name in files:
fullPath = os.path.join(rootdir, name)
fileList.append(fullPath)
fileList.sort()
return fileList
def __getFileAttributes(self, fullPath):
""" get all attributes of the given file,return a attribute dict.
"""
if os.path.exists(fullPath):
type = judgeType(fullPath)
if type == "all":
obj = DataAll(fullPath)
elif type == "others":
obj = Others(fullPath)
elif type == None:
errorMes = "name if %s is not correct" % fullPath
print "cannot get attributes of %s" % fullPath
attributes = {}
return attributes
#raise TypeError(errorMes)
attributes = obj.getAttributes()
#attributes['date'] = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())
else:
attributes = {}
return attributes
def testFunction(self):
result = self.__getFileAttributes(
'/besfs2/offline/data/664-1/jpsi/dst/090613/run_0009952_All_file019_SFO-2.dst'
)
print result
def __registerDir(self, dir):
"""Internal function to register a new directory in DFC .
Returns True for success, False for failure.
"""
fc = self.client
result = fc.createDirectory(dir)
if result['OK']:
if result['Value']['Successful']:
if result['Value']['Successful'].has_key(dir):
return S_OK()
elif result['Value']['Failed']:
if result['Value']['Failed'].has_key(dir):
print 'Failed to create directory %s:%s' % (
dir, result['Value']['Failed'][dir])
return S_ERROR(result)
else:
return S_ERROR(result)
else:
print 'Failed to create directory %s:%s' % (dir, result['Message'])
return S_ERROR(result)
def __registerFileMetadata(self, lfn, attributes):
"""Internal function to set metadata values on a given lfn.
Returns True for success, False for failure.
"""
metadataDict = {}
metadataDict['runL'] = attributes['runL']
metadataDict['runH'] = attributes['runH']
metadataDict['status'] = attributes['status']
metadataDict['eventNumber'] = attributes['eventNumber']
metadataDict['count'] = attributes['count']
result = self.client.setMetadata(lfn, metadataDict)
if not result['OK']:
return S_ERROR()
else:
return S_OK()
def __registerDirMetadata(self, dir, metaDict):
"""Internal function to set metadata to a directory
Returns True for success, False for failure.
"""
fc = self.client
result = fc.setMetadata(dir, metaDict)
if result['OK']:
return S_OK()
else:
message = "Error for setting metadata %s to %s: %s" % (
metaDict, dir, result['Message'])
return S_ERROR('Message')
def __dirExists(self, dir, parentDir):
""" Internal function to check whether 'dir' is the subdirectory of 'parentDir'
Returns 1 for Yes, 0 for NO
"""
fc = self.client
dir_exists = 0
result = fc.listDirectory(parentDir)
if result['OK']:
for i, v in enumerate(
result['Value']['Successful'][parentDir]['SubDirs']):
if v == dir:
dir_exists = 1
break
else:
print 'Failed to list subdirectories of %s:%s' % (
parentDir, result['Message'])
return dir_exists
def __registerSubDirs(self, dirs_dict, dirs_meta):
"""Internal function to create directories in dirs_dict
Returns True for sucess, False for failure
"""
creation_ok = True
for dir in dirs_dict:
if (dir != 'dir_file') & (dir != 'dir_data_mc'):
if self.__registerDir(dirs_meta[dir][0])['OK']:
result = self.__registerDirMetadata(
dirs_meta[dir][0],
{dir.split('_')[1]: dirs_meta[dir][1]})
if not result['OK']:
creation_ok = False
break
else:
print 'Failed to create %s' % dir
creation_ok = False
break
else:
if not self.__registerDir(dirs_meta[dir])['OK']:
print 'Failed to create %s' % dir
creation_ok = False
break
return creation_ok
def registerHierarchicalDir(self, metaDict, rootDir='/bes'):
"""
Create a hierarchical directory according the metadata dictionary
Return created directory for sucess,if this directory has been created, return this existing directory .
Structure of the hierarchical directory:
for real data:/bes/File/resonance/boss version/data/eventType/round
for mc data:/bes/File/resonance/boss version/mc/eventType/round/streamId
The eventType of all real datas is all.
Example:
>>>metaDict = {'dataType': 'dst', 'eventType': 'all', 'streamId': 'stream0','resonance': 'psipp', 'round':'round01','bossVer': '6.6.1'}
>>>badger.registerHierarchicalDir(metaDic)
1
"""
#Save about 20 lines compared with last one
fc = self.client
dir_exists = 0
#0 for failure,1 for success,2 for existing directory
creation_OK = 0
lastDirMetaDict = {
'dataType': metaDict['dataType'],
'streamId': metaDict['streamId']
}
dir_file = rootDir + '/File'
dir_resonance = dir_file + '/' + metaDict['resonance']
dir_bossVer = dir_resonance + '/' + metaDict['bossVer']
if metaDict['streamId'] == 'stream0':
dir_data_mc = dir_bossVer + '/data'
else:
dir_data_mc = dir_bossVer + '/mc'
dir_eventType = dir_data_mc + '/' + metaDict['eventType']
dir_round = dir_eventType + '/' + metaDict['round']
dir_streamId = dir_round + '/' + metaDict['streamId']
# if dir_round has been created,create_round=1
create_round = 0
dirs_dict = [
'dir_file', 'dir_resonance', 'dir_bossVer', 'dir_data_mc',
'dir_eventType', 'dir_round'
]
dirs_meta = {
'dir_file': dir_file,
'dir_data_mc': dir_data_mc,
'dir_resonance': [dir_resonance, metaDict['resonance']],
'dir_bossVer': [dir_bossVer, metaDict['bossVer']],
'dir_eventType': [dir_eventType, metaDict['eventType']],
'dir_round': [dir_round, metaDict['round']]
}
dir_exists = self.__dirExists(dir_file, rootDir)
if not dir_exists:
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_resonance, dir_file)
if not dir_exists:
dirs_dict = dirs_dict[1:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_bossVer, dir_resonance)
if not dir_exists:
dirs_dict = dirs_dict[2:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_data_mc, dir_bossVer)
if not dir_exists:
dirs_dict = dirs_dict[3:]
result = self.__registerSubDirs(dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(dir_eventType,
dir_data_mc)
if not dir_exists:
dirs_dict = dirs_dict[4:]
result = self.__registerSubDirs(
dirs_dict, dirs_meta)
if result:
create_round = 1
else:
dir_exists = self.__dirExists(
dir_round, dir_eventType)
if not dir_exists:
dirs_dict = dirs_dict[5:]
result = self.__registerSubDirs(
dirs_dict, dirs_meta)
if result:
create_round = 1
else:
create_round = 1
if create_round:
if metaDict['streamId'] != "stream0":
dir_exists = self.__dirExists(dir_streamId, dir_round)
if not dir_exists:
if self.__registerDir(dir_streamId)['OK']:
result = self.__registerDirMetadata(
dir_streamId, {'streamId': metaDict['streamId']})
if result['OK']:
result = self.__registerDirMetadata(
dir_streamId, lastDirMetaDict)
if result['OK']:
creation_OK = 1
else:
creation_OK = 2
else:
result = self.__registerDirMetadata(dir_round, lastDirMetaDict)
if result['OK']:
creation_OK = 1
if (creation_OK == 1) | (creation_OK == 2):
if metaDict['streamId'] == "stream0":
return dir_round
else:
return dir_streamId
##########################################################################################
#dir options
def removeDir(self, dir):
"""remove the dir include files and subdirs
"""
result = self.client.listDirectory(dir)
if result['OK']:
if not result['Value']['Successful'][dir]['Files'] and not result[
'Value']['Successful'][dir]['SubDirs']:
#print 'no file and subDirs in this dir'
self.client.removeDirectory(dir)
return S_OK()
else:
if result['Value']['Successful'][dir]['Files']:
for file in result['Value']['Successful'][dir]['Files']:
self.client.removeFile(file)
else:
for subdir in result['Value']['Successful'][dir][
'SubDirs']:
self.removeDir(subdir)
self.removeDir(dir)
def listDir(self, dir):
"""list the files under the given DFC dir"""
fileList = []
result = self.client.listDirectory(dir)
if result['OK']:
if result['Value']['Successful'][dir]['Files']:
fileList = result['Value']['Successful'][dir]['Files'].keys()
fileList.sort()
else:
print "no files under this dir"
return fileList
def getDirMetaVal(self, dir):
"""list the registed metadata value of the given dir"""
result = self.client.getDirectoryMetadata(dir)
if result['OK']:
return result['Value']
else:
print "Failed to get meta Value of the directory"
return {}
#################################################################################
# meta fields operations
#
def addNewFields(self, fieldName, fieldType, metaType='-d'):
"""add new fields,if metaType is '-f',add file field,
fileType is datatpye in MySQL notation
"""
result = self.client.addMetadataField(fieldName, fieldType, metaType)
if not result['OK']:
return S_ERROR(result)
else:
return S_OK()
def deleteMetaField(self, fieldName):
"""delete a exist metafield"""
result = self.client.deleteMetadataField(fieldName)
if not result['OK']:
return S_ERROR(result)
else:
return S_OK()
def getAllFields(self):
"""get all meta fields,include file metafield and dir metafield.
"""
result = self.client.getMetadataFields()
if not result['OK']:
return S_ERROR(result['Message'])
else:
return result['Value']
def registerFileMetadata(self, lfn, metaDict):
"""Add file level metadata to an entry
True for success, False for failure
(maybe used to registerNewMetadata
Example:
>>>lfn = '/bes/File/psipp/6.6.1/data/all/exp1/run_0011414_All_file001_SFO-1'
>>>entryDict = {'runL':1000,'runH':898898}
>>>badger.registerFileMetadata(lfn,entryDict)
True
"""
fc = self.client
result = fc.setMetadata(lfn, metaDict)
if result['OK']:
return S_OK()
else:
print 'Error:%s' % (result['Message'])
return S_ERROR(result['Message'])
#####################################################################
# File Options
def registerFile(self, lfn, dfcAttrDict):
"""Register a new file in the DFC.
"""
#TODO:need more tests,if directory of file doesn't exist,
#addFile will create it without setting any metadata(lin lei)
#need to check whether directory of file exists in dfc?(lin lei)
#pass
fc = self.client
result = fc.addFile({lfn: dfcAttrDict})
if result['OK']:
if result['Value']['Successful']:
if result['Value']['Successful'].has_key(lfn):
return S_OK()
elif result['Value']['Failed']:
if result['Value']['Failed'].has_key(lfn):
print 'Failed to add this file:', result['Value'][
'Failed'][lfn]
return S_ERROR()
else:
print 'Failed to add this file :', result['Message']
return S_ERROR()
# need to register file (inc. creating appropriate directory
# if it doesn't already exist; and register metadata for that
# file / directory
# Q: how / where to pass the metadata?
def getFileMetaVal(self, lfn):
"""get the File Meta Value of given file
"""
result = self.client.getFileUserMetadata(lfn)
if result['OK']:
return result['Value']
else:
print "Failed to get meta Value of this file"
return {}
def reCalcCount(self, fileList, plus=True):
"""calculate the value of metadata 'count',when a file contain in a dataset
count+1,when del a dataset,then all file in this dataset count -1
default plus=True,means count+1,if count-1,set plus=False
return the value of count, count = -1 means error.
NOTE:this function should only be called when create or delete a dataset.
"""
countDict = {}
if type(fileList) != type([]):
fileList = [fileList]
for file in fileList:
result = self.getFileMetaVal(file)
if len(result) != 0:
count = result['count']
if plus:
count += 1
else:
if count > 0:
count -= 1
cDict = {'count': count}
self.registerFileMetadata(file, cDict)
countDict[file] = count
else:
print "Failed reCalculate value of count of file %s" % file
countDict[file] = -1
return countDict
def removeFile(self, lfn):
"""remove file on DFC
"""
result = self.client.removeFile(lfn)
if not result['OK']:
return S_ERROR(result)
else:
return S_OK()
def getPFN(self, lfn):
"""get replicas by lfn"""
result = self.client.getReplicas(lfn)
#print result
if not result['OK']:
return S_ERROR(result)
else:
return S_OK(result['Value']['Successful'][lfn]['IHEPD-USER'])
def getSize(self, lfns):
"""get the size of the given lfn"""
result = self.client.getFileSize(lfns)
if result['OK']:
if result['Value']['Successful']:
retVal = result['Value']['Successful']
else:
retVal = {}
return retVal
def getFilesByMetadataQuery(self, query):
"""Return a list of LFNs satisfying given query conditions.
Example usage:
>>> brunH_GT_29756adger.getFilesByMetadataQuery('resonance=jpsi bossVer=6.5.5 round=exp1')
['/bes/File/jpsi/6.5.5/data/all/exp1/file1', .....]
"""
#TODO: checking of output, error catching
fc = self.client
#TODO: calling the FileCatalog CLI object and its private method
# is not a good way of doing this! but use it to allow construction of
# the query meantime, until createQuery is made a public method
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(query)
result = fc.findFilesByMetadata(metadataDict, '/')
if result['OK']:
lfns = fc.findFilesByMetadata(metadataDict, '/')['Value']
lfns.sort()
return lfns
else:
print "ERROR: No files found which match query conditions."
return None
def uploadAndRegisterFiles(self,
fileList,
SE='IHEPD-USER',
guid=None,
ePoint=''):
"""upload a set of files to SE and register it in DFC.
user input the directory of localfile.
argument:
ePoint is the energy point,for scan data
we can treat localDir as a kind of datasetName.
"""
result_OK = 1
errorList = []
#fileList = self.getFilenamesByLocaldir(localDir)
for fullpath in fileList:
#get the attributes of the file
fileAttr = self.__getFileAttributes(fullpath)
if len(fileAttr) == 0:
print "failed to get file %s attributes" % fullpath
return S_ERROR("failed to get file attributes")
#create dir and set dirMetadata to associated dir
lastDir = self.registerHierarchicalDir(fileAttr, rootDir='/bes')
dirMeta = self.getDirMetaVal(lastDir)
if not (dirMeta.has_key("jobOptions")
or dirMeta.has_key("description")):
lastDirMetaDict = {}
lastDirMetaDict['jobOptions'] = fileAttr['jobOptions']
lastDirMetaDict['description'] = fileAttr['description']
try:
self.__registerDirMetadata(lastDir, lastDirMetaDict)
except:
pass
if len(ePoint):
lastDir = lastDir + os.sep + ePoint
lfn = lastDir + os.sep + fileAttr['LFN']
#upload and register file.
dirac = Dirac()
result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True)
#register file metadata
if not result['OK']:
print 'ERROR %s' % (result['Message'])
#return S_ERROR(result['Message'])
errorList.append(fullpath)
result_OK = 0
else:
result = self.__registerFileMetadata(lfn, fileAttr)
if not result['OK']:
result_OK = 0
print "failed to register file metadata"
if result_OK:
return S_OK()
else:
return S_ERROR(errorList)
def downloadFilesByFilelist(self, fileList, destDir=''):
"""downLoad a set of files form SE.
use getFilesByFilelist() get a list of lfns and download these files.
fileList get from function getFilesByDatesetName()
Example usage:
>>>badger.downloadFilesByFilelist(fileList)
"""
errorDict = {}
dirac = Dirac()
#fileList = self.getFilesByDatasetName(dataset_name)
for lfn in fileList:
result = dirac.getFile(lfn, destDir, printOutput=False)
if not result['OK']:
errorDict[lfn] = result['Message']
if errorDict:
serr = S_ERROR()
serr["errorDict"] = errorDict
return serr
else:
return S_OK("File download successfully.")
####################################################################
# dataset functions
#
def registerDataset(self, datasetName, path, conditions):
"""Register a new dataset in DFC. Takes dataset name and string with
conditions for new dataset as arguments.
datasetname format:
resonance_BossVer_eventtype_round_runL_runH_stream0_datatype
resonance_BossVer_eventtype_round_runL_runH_streamID_datatype
type(conditions) is str,like "resonance=jpsi bossVer=655 round=round1"
"""
fc = self.client
cli = FileCatalogClientCLI(fc)
metadataDict = cli._FileCatalogClientCLI__createQuery(conditions)
metadataDict['Path'] = path
result = fc.addDataset(datasetName, metadataDict)
if not result['OK']:
print("Error: %s" % result['Message'])
return S_ERROR()
else:
print "Added dataset %s with conditions %s" % (datasetName,
conditions)
return S_OK()
def getDatasetDescription(self, datasetName):
"""Return a string containing a description of metadata with which
the given dataset was defined.
Example usage:
>>> result = badger.getDatasetDescription('psipp_661_data_all_exp2')
"""
result = self.client.getDatasetParameters(datasetName)
if not result['OK']:
print "ERROR: failed to get status of dataset:", result['Message']
else:
parDict = result['Value']
for par, value in parDict.items():
print par.rjust(20), ':', value
def getDatasetMetadata(self, datasetName):
"""Return a dict containing a description of metadata with which
the given dataset was defined.
Example usage:
>>> result = badger.getDatasetMetadata('psipp_661_data_all_exp2')
"""
result = self.client.getDatasetParameters(datasetName)
if not result['OK']:
print "ERROR: failed to get status of dataset:", result['Message']
return S_ERROR(result['Message'])
else:
parDict = result['Value']
return S_OK(parDict)
def removeDataset(self, datasetName):
"""remove a dataset
"""
result = self.client.removeDataset(datasetName)
if not result['OK']:
print "ERROR: failed to remove dataset:", result['Message']
else:
print "Successfully removed dataset", datasetName
def checkDataset(self, datasetName):
""" check if the dataset parameters are still valid
"""
result = self.client.checkDataset(datasetName)
if not result['OK']:
print "ERROR: failed to check dataset:", result['Message']
else:
changeDict = result['Value']
if not changeDict:
print "Dataset is not changed"
else:
print "Dataset changed:"
for par in changeDict:
print " ", par, ': ', changeDict[par][
0], '->', changeDict[par][1]
def updateDataset(self, datasetName):
""" Update the given dataset parameters
"""
result = self.client.updateDataset(datasetName)
if not result['OK']:
print "ERROR: failed to update dataset:", result['Message']
else:
print "Successfully updated dataset", datasetName
def freezeDataset(self, datasetName):
""" Freeze the given dataset
"""
result = self.client.freezeDataset(datasetName)
if not result['OK']:
print "ERROR: failed to freeze dataset:", result['Message']
else:
print "Successfully frozen dataset", datasetName
def releaseDataset(self, datasetName):
""" Release the given dataset
"""
result = self.client.releaseDataset(datasetName)
if not result['OK']:
print "ERROR: failed to release dataset:", result['Message']
else:
print "Successfully released dataset", datasetName
def getFilesByDatasetName(self, datasetName):
"""Return a list of LFNs in the given dataset.
Example usage:
>>> badger.getFilesByDatasetName('psipp_661_data_all_exp2')
['/bes/File/psipp/6.6.1/data/all/exp2/file1', .....]
"""
fc = self.client
result = returnSingleResult(fc.getDatasetFiles(datasetName))
if result['OK']:
lfns = result['Value']
lfns.sort()
return S_OK(lfns)
else:
print "ERROR: Dataset", datasetName, " not found"
return S_ERROR(result)
def listDatasets(self):
"""list the exist dataset"""
datasetName = ''
result = self.client.getDatasets(datasetName)
if not result['OK']:
print "ERROR:failed to get datasets"
return
datasetDict = result['Value']
for dName in datasetDict.keys():
print dName
def checkDatasetIntegrity():
pass
