def deleteEntries(pLfnList, pKeepLfn, pRemoveLinks, pVerbose):
"""
For each LFN in pLfnList, the contents of LFC_HOME env var are prepended
for the relative paths (absoulte paths are used as they are), and
deleteOneEntry is invoked.
Please check deleteOneEntry for the meaning of the other arguments.
"""
lfnList = pLfnList
keepLfn = pKeepLfn
removeLinks = pRemoveLinks
verbose = pVerbose
rc = 0
for lfn in lfnList:
lfn = lfn.strip()
[lfn, dir] = checkHomeDir(lfn)
err = lfc.lfc_chdir(dir)
if (err < 0):
sys.stderr.write("Error changing dir to: " + dir + " : " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
return err
err = deleteOneEntry(lfn, dir, keepLfn, removeLinks, verbose)
if (err): rc = err
# Return the error code
return rc
def deleteEntries(pLfnList, pKeepLfn, pRemoveLinks, pVerbose):
"""
For each LFN in pLfnList, the contents of LFC_HOME env var are prepended
for the relative paths (absoulte paths are used as they are), and
deleteOneEntry is invoked.
Please check deleteOneEntry for the meaning of the other arguments.
"""
lfnList = pLfnList
keepLfn = pKeepLfn
removeLinks = pRemoveLinks
verbose = pVerbose
rc = 0
for lfn in lfnList:
lfn = lfn.strip()
[lfn, dir] = checkHomeDir(lfn)
err = lfc.lfc_chdir(dir)
if(err < 0):
sys.stderr.write("Error changing dir to: "+dir+" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
return err
err = deleteOneEntry(lfn, dir, keepLfn, removeLinks, verbose)
if(err): rc=err
# Return the error code
return rc
def readdirxr_recurse(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
#---------------------------------------------------------------------------
# Loop on the entries of the folder
#---------------------------------------------------------------------------
folders = []
while 1:
read_pt = lfc.lfc_readdirxr(dir, '')
if read_pt == None:
break
entry, replicas = read_pt
name = prefix + entry.d_name
if entry.filemode & 040000:
folders.append(name)
comment = ' (Folder)'
elif entry.nbreplicas == 0:
comment = ' (No replica)'
else:
comment = ''
print name + comment
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas
#-------------------------------------------------------------------------
if comment == '':
for replica in replicas:
print ' ==>', replica.sfn
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Recurse on subfolders
#---------------------------------------------------------------------------
for name in folders:
readdirxr_recurse(name)
def readdirxr(folderFiles):
folderKeys = folderFiles.keys()
folderKeys.sort()
#print folderKeys
#---------------------------------------------------------------------------
# Loop on the folders
#---------------------------------------------------------------------------
for folder in folderKeys:
prefix = folder
#print '\n' + prefix + ':', folderFiles[prefix]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#-------------------------------------------------------------------------
# Open the folder
#-------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
#-------------------------------------------------------------------------
# Loop on the entries of the folder which are in the dictionary
#-------------------------------------------------------------------------
while 1:
read_pt = lfc.lfc_readdirxr(dir, '')
if read_pt == None:
break
entry, replicas = read_pt
if entry.d_name not in folderFiles[prefix]:
break
if entry.filemode & 040000:
comment = ' (Folder)'
elif entry.nbreplicas == 0:
comment = ' (No replica)'
else:
comment = ''
print prefix + entry.d_name
#-----------------------------------------------------------------------
# If the entry is a regular file, list its replicas
#-----------------------------------------------------------------------
if comment == '':
for replica in replicas:
print ' ==>', replica.sfn
lfc.lfc_closedir(dir)
def delUser(userDN, _userid):
res = lfc.lfc_rmusrmap(0, userDN)
if res == 0:
print "User has been removed from userinfo"
return 0
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
print "There was an error while deleting user: Error " + str(
err_num) + " (" + err_string + ")"
return -1
def readdirg_recurse(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
folderFiles = {prefix: []}
#---------------------------------------------------------------------------
# Append the entries of the folder to the current list
#---------------------------------------------------------------------------
files = []
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
files.append({'name': entry.d_name,
'mode': entry.filemode})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the entries of the current file list to build the dictionary
#---------------------------------------------------------------------------
for myfile in files:
folderFiles[prefix].append(myfile['name'])
#-------------------------------------------------------------------------
# If the entry is a folder, recurse
#-------------------------------------------------------------------------
if myfile['mode'] & 040000:
folderFiles.update(readdirg_recurse(prefix + myfile['name']))
return folderFiles
def rmDir(nickname):
dirname = "/grid/lhcb/user/" + nickname[0] + "/" + nickname
res = lfc.lfc_rmdir(dirname)
if res == 0:
print "user dir has been removed "
return 0
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
print "There was an error while deleting dir: Error " + str(
err_num) + " (" + err_string + ")"
return -1
def deleteOneReplica(pSfn, pVerbose):
"""
Tries to delete the replica information with the SFN specified.
"""
verbose = pVerbose
if(verbose):
print "--lfc.lfc_delreplica(\"\", None, \""+pSfn+"\")"
if(lfc.lfc_delreplica("", None, pSfn) < 0):
sys.stderr.write("Warning: Error deleting replica:"+pSfn+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
return (-1)
def updateDirPerm(userid, nickname, userDN):
dirname = "/grid/lhcb/user/" + nickname[0] + "/" + nickname
print "updating owner of user dir to " + str(userid)
res = lfc.lfc_chown(dirname, userid, -1)
if res == 0:
print "Owner directory updated"
return 0
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
print "There was an error while updating ownerid for " + dirname + " Error " + str(
err_num) + " (" + err_string + ")"
return -1
def deleteOneReplica(pSfn, pVerbose):
"""
Tries to delete the replica information with the SFN specified.
"""
verbose = pVerbose
if (verbose):
print "--lfc.lfc_delreplica(\"\", None, \"" + pSfn + "\")"
if (lfc.lfc_delreplica("", None, pSfn) < 0):
sys.stderr.write("Warning: Error deleting replica:" + pSfn + ": " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
return (-1)
def deleteOneLFN(pLfn, pVerbose):
"""
Tries to delete the LFN (or sym link) specified
"""
lfn = pLfn
verbose = pVerbose
if (verbose):
print "--lfc.lfc_unlink(\"" + lfn + "\")"
if (lfc.lfc_unlink(lfn) < 0):
sys.stderr.write("Warning: Error removing LFN:" + lfn + ": " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
return -1
def deleteOneLFN(pLfn, pVerbose):
"""
Tries to delete the LFN (or sym link) specified
"""
lfn=pLfn
verbose = pVerbose
if(verbose):
print "--lfc.lfc_unlink(\""+lfn+"\")"
if(lfc.lfc_unlink(lfn)<0):
sys.stderr.write("Warning: Error removing LFN:"+lfn+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
return -1
def createDir(dirname, _nickname):
res = lfc.lfc_mkdir(dirname, 0777)
if res == 0:
print "Directory " + dirname + " has been created and " + str(res)
#we delete the directory
#res = lfc.lfc_rmdir(dirname)
return -1
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
print "There was an error while creating " + dirname + " Error " + str(
err_num) + " (" + err_string + ")"
return -2
def getUserName(userID):
result, u_list = lfc.lfc_getusrmap()
if (result == 0):
for i in u_list:
if i.userid == userID:
print "userID exists mapped to " + str(i.username)
return i.username
print "Did not find the corresponding username for " + str(userID)
return "ERROR"
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
print "There was an error while looking at username for " + str(
userID) + " Error " + str(err_num) + " (" + err_string + ")"
return "ERROR"
def readdirg_recurse(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
files = []
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder to build the list of files
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
files.append({'name': prefix + entry.d_name,
'mode': entry.filemode,
'guid': entry.guid})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the current file list : If the entry is a folder, recurse
#---------------------------------------------------------------------------
for myfile in files[:]: # Local copy is mandatory
if myfile['mode'] & 040000:
files.extend(readdirg_recurse(myfile['name']))
return files
def createUser(userDN, userid):
res = lfc.lfc_enterusrmap(userid, userDN)
if res == 0:
print "User created " + userDN
return 0
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
# error not due to userDN exists
if err_num != 17:
print "There was an error while adding " + userDN + " Error " + str(
err_num) + " (" + err_string + ")"
return -1
#however we return 0 because we want to update the dir as it has been created by root
else:
print "UserDN already exists"
return 0
def checkDir(nickname):
first_letter = nickname[0]
dirname = '/grid/lhcb/user/'
dirname += first_letter
dirname += '/'
dirname += nickname
stat = lfc.lfc_filestatg()
res = lfc.lfc_statg(dirname, "", stat)
if res == 0:
print "Directory exists "
return 0
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
print "There was an error while looking for " + dirname + ": Error " + str(
err_num) + " (" + err_string + ")"
if err_num == 2:
res = createDir(dirname, nickname)
return res
else:
return -3
def readdirg(*args):
if len(args) < 1:
folder = ''
else:
folder = args[0]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
#---------------------------------------------------------------------------
# Loop on the entries of the folder
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if entry.filemode & 040000:
marker = '/'
elif entry.filemode & 020000: # not entry.guid:
marker = '@'
else:
marker = ''
print ('%06o' % entry.filemode) + ' ' + entry.d_name + marker
lfc.lfc_closedir(dir)
def del_replicas(sfns,guids):
rem_guids=[]
rem_sfns=[]
for sfn_prefix in SFN_PREFIXES:
prefixed_sfns=[ sfn_prefix + sfn for sfn in sfns ]
result,err_nums=lfc.lfc_delreplicasbysfn(prefixed_sfns,guids)
#print err_nums
for i in range(len(err_nums)):
err_num=err_nums[i]
if err_num == 2:
rem_guids.append(guids[i])
rem_sfns.append(sfns[i])
elif err_num != 0:
err_string = lfc.sstrerror(err_num)
print "There was an error for guid " + guids[i] + ": " + str(err_num) + " (" + err_string + ")"
guids=rem_guids
sfns=rem_sfns
rem_guids=[]
rem_sfns=[]
if len(guids) < 0:
break
for guid in guids:
print "There was an error for guid " + guid + ": 2 (No such file or directory), Correct SFN may not be in list or file already deleted."
def deleteOneDir(pDir, pForce, pExtLinks, pExtLfns, pVerbose):
"""
Tries to delete the specified directory. If "pForce" is true, then the
directory will be removed even if it is not empty and all the entries in
the directory (including subdirectories) will be removed as well, in a
recursive way. Otherwise, the deletion will be performed only if the
directory is empty.
If pExtLinks == pExtLfns == False, the function will remove only the
specified directory contents and will not to touch external sym links
pointing to LFNs in the directory, nor entries pointed to by links located
in the directory.
If pExtLinks == True, then links located in other directories that point
to LFNs in the specified directory are also removed.
If pExtLfns == True, then LFNs (and their replicas and all their sym links)
located in other directories and pointed to by links in the specified
directory are also removed.
"""
dir = pDir
force = pForce
extLinks = pExtLinks
extLfns = pExtLfns
verbose = pVerbose
rc = 0
err = 0
subdirlist = [] # Subdirectories to remove when we are done with current one
[dir, parentdir] = lfc_del_lfn.checkHomeDir(dir)
fulldir = parentdir+dir
if(verbose):
print "--Deleting Dir: "+dir
err = lfc.lfc_chdir(fulldir)
if(err < 0):
sys.stderr.write("Error changing dir to: "+fulldir+" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
return err
dir_p=lfc.lfc_DIR()
dir_entry=lfc.lfc_direnstatg()
dir_p=lfc.lfc_opendirg(".", "")
if(dir_p < 0):
sys.stderr.write("Error opening specified dir: "+dir+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
return -1
# Read first entry
lfc.lfc_rewinddir(dir_p)
dir_entry=lfc.lfc_readdirg(dir_p)
if(force):
# Remove all LFNs in a loop
S_IFDIR = 0x4000
S_IFLNK = 0xA000
while(dir_entry):
lfn = dir_entry.d_name
if(dir_entry.filemode & S_IFDIR):
# This entry is a directory
subdirlist.append(lfn)
else:
if((dir_entry.filemode & S_IFLNK) == S_IFLNK):
# This entry is a sym link
if(extLfns):
# Remove the replicas and all the links (including main LFN)
err = lfc_del_lfn.deleteOneEntry(lfn, fulldir, False, True, verbose)
else:
# Remove only the sym link (no replicas)
err = lfc_del_lfn.deleteOneLFN(lfn, verbose)
else:
# This entry is a main LFN
# # First check if the file has been alredy deleted (links, etc...)
# fstat = lfc.lfc_filestatg()
# if(lfc.lfc_statg(lfn, "", fstat)<0):
# if(verbose):
# print "--Warning. Skipping deletion of non-accessible file:",lfn,":",\
# lfc.sstrerror(lfc.cvar.serrno)
# else:
if(extLinks):
# Remove the replicas and all the links that point to this LFN
err = lfc_del_lfn.deleteOneEntry(lfn, fulldir, False, True, verbose)
else:
# Remove only this LFN and replicas (but no external sym links)
err = lfc_del_lfn.deleteOneEntry(lfn, fulldir, False, False, verbose)
if(err): rc = err
dir_entry=lfc.lfc_readdirg(dir_p)
else:
if(dir_entry):
sys.stderr.write("Error: Directory "+dir+" not empty! Consider using -r.\n")
return -1
# Close the directory
if(lfc.lfc_closedir(dir_p) < 0):
sys.stderr.write("Error closing dir: "+dir+" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
# Remove all subdirectories in the list
for subdir in subdirlist:
err = deleteOneDir(fulldir+'/'+subdir, force, extLinks, extLfns, verbose)
if(err): rc=err
# Finally, remove also the top directory itself
err = lfc.lfc_chdir("..")
if(err < 0):
sys.stderr.write("Error changing dir to \"..\" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
return err
if(verbose):
print "--lfc.lfc_unlink(\""+dir+"\")"
err = lfc.lfc_rmdir(dir)
if(err<0):
sys.stderr.write("Error removing dir: "+dir+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n")
return err
# Return the error code
# return rc
return err
def readdirg_lr_recurse_timestamp(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
time_format = '%Y/%m/%d %H:%M:%S %Z'
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
name = prefix + entry.d_name
sys.stdout.write(time.strftime(time_format) + ' ' + name + "\n")
sys.stdout.flush()
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (entry.filemode & 060000): # Exclude folders and symbolic links
flag = lfc.CNS_LIST_BEGIN
while 1:
res = lfc.lfc_listreplica('', entry.guid, flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
sys.stdout.write(time.strftime(time_format) + ' ==> ' + res.sfn + "\n")
sys.stdout.flush()
lfc.lfc_listreplica('', entry.guid, lfc.CNS_LIST_END, listp)
#-------------------------------------------------------------------------
# If the entry is a folder, recurse
#-------------------------------------------------------------------------
if entry.filemode & 040000:
readdirg_lr_recurse_timestamp(name)
lfc.lfc_closedir(dir)
file.sfn = "srm://example.com/path/"
file.server = "example.com"
file.csumtype = "MD"
file.csumvalue = "1a31009319c99ebdefd23055b20ff034"
farray = [file,]
retCode = 0
# Remove if it was already there
lfc.lfc_delreplicasbysfn([file.sfn,], [""])
# Register
print "Registering file"
if lfc.lfc_registerfiles(farray)[0] != 0:
print "Failed to register the file: %s" % lfc.sstrerror(lfc.cvar.serrno)
sys.exit(1)
# Do the lfc_statr and check
statg = lfc.lfc_filestatg()
if lfc.lfc_statr(file.sfn, statg) != 0:
print "Failed to do the lfc_statr: %s" % lfc.sstrerror(lfc.cvar.serrno)
retCode = 1
if statg.csumtype != file.csumtype or statg.csumvalue != file.csumvalue or statg.guid != file.guid:
print "Cns_statr repbuffer too small!"
retCode = 1
else:
print "Test for lfc_statr passed!!"
# Try with statg
def _getPFNsLFC(guids,lfcHost,storages,nFiles,verbose=False):
# set LFC HOST
os.environ['LFC_HOST'] = lfcHost
# timeout
os.environ['LFC_CONNTIMEOUT'] = '60'
os.environ['LFC_CONRETRY'] = '2'
os.environ['LFC_CONRETRYINT'] = '6'
if verbose:
print "Get file info from %s" % lfcHost
# get PFN
iGUID = 0
nGUID = 10000
pfnMap = {}
listGUID = []
for guid in guids:
iGUID += 1
listGUID.append(guid)
if iGUID % nGUID == 0 or iGUID == len(guids):
sys.stdout.write('.')
sys.stdout.flush()
# get replica
nTry = 3
for iTry in range(nTry):
ret,resList = lfc.lfc_getreplicas(listGUID,'')
if ret != 0 and iTry+1<nTry:
print "retry to access %s %s/%s" % (lfcHost,iTry,nTry)
time.sleep(30)
else:
break
if ret == 0:
for fr in resList:
if fr != None and ((not hasattr(fr,'errcode')) or \
(hasattr(fr,'errcode') and fr.errcode == 0)):
# get host
match = re.search('[^:]+://([^:/]+):*\d*/',fr.sfn)
if match==None:
continue
# check host
host = match.group(1)
if storages != [] and (not host in storages):
continue
# skip tape
onTapeFlag = False
for tapePath in ['/MCTAPE/','/BNLT1D0/','/atlasmctape/','/atlasdatatape/',
'/castor/cern.ch/grid/atlas/tzero/',
'/castor/cern.ch/grid/atlas/DAQ/']:
if re.search(tapePath,fr.sfn) != None:
onTapeFlag = True
break
if onTapeFlag:
continue
# append
if not pfnMap.has_key(fr.guid):
pfnMap[fr.guid] = []
pfnMap[fr.guid].append(fr.sfn)
else:
print "ERROR : %s" % lfc.sstrerror(lfc.cvar.serrno)
sys.exit(EC_LFC)
# reset
listGUID = []
# break
if nFiles > 0 and len(pfnMap) >= nFiles:
break
# return
return pfnMap
def print_error(self, msg):
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
print >> sys.stderr, msg, "Error " + str(err_num) + " (" + err_string + ")"
return err_num
def list_replicas(*names):
#---------------------------------------------------------------------------
# Loop on the file names given as parameters
#---------------------------------------------------------------------------
for name in names:
if name[0] != '/':
if 'LFC_HOME' in os.environ:
name = os.environ['LFC_HOME'] + '/' + name
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#-------------------------------------------------------------------------
# stat an existing entry in the LFC and print the GUID
#-------------------------------------------------------------------------
statg = lfc.lfc_filestatg()
res = lfc.lfc_statg(name, '', statg)
if res != 0:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' while looking for ' + name + ': ' + err_string)
if statg.filemode & 040000:
print ('%06o' % statg.filemode), name, 'is a folder'
guid = statg.guid
if guid:
print ('%06o' % statg.filemode), name, 'has guid:' + guid
else:
print ('%06o' % statg.filemode), name, 'has NO guid'
#-------------------------------------------------------------------------
# retrieve the comment on a file
#-------------------------------------------------------------------------
buffer = ' ' * lfc.CA_MAXCOMMENTLEN
res = lfc.lfc_getcomment(name, buffer)
if res != 0:
err_num = lfc.cvar.serrno
if err_num != 2:
err_string = lfc.sstrerror(err_num)
print 'Error ' + str(err_num) + ' while reading the comment for ' + name + ': ' + err_string
else:
print "Comment: '" + buffer.rstrip(' ') + "'"
#-------------------------------------------------------------------------
# list the replicas of a given entry, starting from the GUID
#-------------------------------------------------------------------------
listp = lfc.lfc_list()
flag = lfc.CNS_LIST_BEGIN
num_replicas = 0
while 1:
res = lfc.lfc_listreplica('', guid, flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
print ' ==>', res.sfn
num_replicas += 1
lfc.lfc_listreplica('', guid, lfc.CNS_LIST_END, listp)
print 'Found ' + str(num_replicas) + ' replica(s)\n'
def readdirg_lr_recurse(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Append the entries of the folder to the current list
#---------------------------------------------------------------------------
files = []
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
files.append({'name': prefix + entry.d_name,
'mode': entry.filemode,
'guid': entry.guid})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the entries of the current file list
#---------------------------------------------------------------------------
for myfile in files:
print myfile['name']
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (myfile['mode'] & 060000): # Exclude folders and symbolic links
flag = lfc.CNS_LIST_BEGIN
while 1:
res = lfc.lfc_listreplica('', myfile['guid'], flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
print ' ==>', res.sfn
lfc.lfc_listreplica('', myfile['guid'], lfc.CNS_LIST_END, listp)
#-------------------------------------------------------------------------
# If the entry is a folder, recurse
#-------------------------------------------------------------------------
if myfile['mode'] & 040000:
readdirg_lr_recurse(myfile['name'])
def readdirg_gr(*args):
if len(args) < 1:
folder = ''
else:
folder = args[0]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
files = []
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder to build the list of files
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if entry.filemode & 040000:
marker = '/'
elif entry.filemode & 020000: # not entry.guid:
marker = '@'
else:
marker = ''
files.append({'name': entry.d_name + marker,
'mode': entry.filemode,
'guid': entry.guid})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the list of files
#---------------------------------------------------------------------------
lfc.lfc_startsess('', '')
for myfile in files:
print ('%06o' % myfile['mode']) + ' ' + myfile['name']
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (myfile['mode'] & 060000): # Exclude folders and symbolic links
(res, replicas) = lfc.lfc_getreplica('', myfile['guid'], '')
if res == 0:
for replica in replicas:
print ' ==>', replica.sfn
print 'Found ' + str(len(replicas)) + ' replica(s)'
print
lfc.lfc_endsess()
def deleteOneDir(pDir, pForce, pExtLinks, pExtLfns, pVerbose):
"""
Tries to delete the specified directory. If "pForce" is true, then the
directory will be removed even if it is not empty and all the entries in
the directory (including subdirectories) will be removed as well, in a
recursive way. Otherwise, the deletion will be performed only if the
directory is empty.
If pExtLinks == pExtLfns == False, the function will remove only the
specified directory contents and will not to touch external sym links
pointing to LFNs in the directory, nor entries pointed to by links located
in the directory.
If pExtLinks == True, then links located in other directories that point
to LFNs in the specified directory are also removed.
If pExtLfns == True, then LFNs (and their replicas and all their sym links)
located in other directories and pointed to by links in the specified
directory are also removed.
"""
dir = pDir
force = pForce
extLinks = pExtLinks
extLfns = pExtLfns
verbose = pVerbose
rc = 0
err = 0
subdirlist = [
] # Subdirectories to remove when we are done with current one
[dir, parentdir] = lfc_del_lfn.checkHomeDir(dir)
fulldir = parentdir + dir
if (verbose):
print "--Deleting Dir: " + dir
err = lfc.lfc_chdir(fulldir)
if (err < 0):
sys.stderr.write("Error changing dir to: " + fulldir + " : " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
return err
dir_p = lfc.lfc_DIR()
dir_entry = lfc.lfc_direnstatg()
dir_p = lfc.lfc_opendirg(".", "")
if (dir_p < 0):
sys.stderr.write("Error opening specified dir: " + dir + ": " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
return -1
# Read first entry
lfc.lfc_rewinddir(dir_p)
dir_entry = lfc.lfc_readdirg(dir_p)
if (force):
# Remove all LFNs in a loop
S_IFDIR = 0x4000
S_IFLNK = 0xA000
while (dir_entry):
lfn = dir_entry.d_name
if (dir_entry.filemode & S_IFDIR):
# This entry is a directory
subdirlist.append(lfn)
else:
if ((dir_entry.filemode & S_IFLNK) == S_IFLNK):
# This entry is a sym link
if (extLfns):
# Remove the replicas and all the links (including main LFN)
err = lfc_del_lfn.deleteOneEntry(
lfn, fulldir, False, True, verbose)
else:
# Remove only the sym link (no replicas)
err = lfc_del_lfn.deleteOneLFN(lfn, verbose)
else:
# This entry is a main LFN
# # First check if the file has been alredy deleted (links, etc...)
# fstat = lfc.lfc_filestatg()
# if(lfc.lfc_statg(lfn, "", fstat)<0):
# if(verbose):
# print "--Warning. Skipping deletion of non-accessible file:",lfn,":",\
# lfc.sstrerror(lfc.cvar.serrno)
# else:
if (extLinks):
# Remove the replicas and all the links that point to this LFN
err = lfc_del_lfn.deleteOneEntry(
lfn, fulldir, False, True, verbose)
else:
# Remove only this LFN and replicas (but no external sym links)
err = lfc_del_lfn.deleteOneEntry(
lfn, fulldir, False, False, verbose)
if (err): rc = err
dir_entry = lfc.lfc_readdirg(dir_p)
else:
if (dir_entry):
sys.stderr.write("Error: Directory " + dir +
" not empty! Consider using -r.\n")
return -1
# Close the directory
if (lfc.lfc_closedir(dir_p) < 0):
sys.stderr.write("Error closing dir: " + dir + " : " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
# Remove all subdirectories in the list
for subdir in subdirlist:
err = deleteOneDir(fulldir + '/' + subdir, force, extLinks, extLfns,
verbose)
if (err): rc = err
# Finally, remove also the top directory itself
err = lfc.lfc_chdir("..")
if (err < 0):
sys.stderr.write("Error changing dir to \"..\" : " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
return err
if (verbose):
print "--lfc.lfc_unlink(\"" + dir + "\")"
err = lfc.lfc_rmdir(dir)
if (err < 0):
sys.stderr.write("Error removing dir: " + dir + ": " +
lfc.sstrerror(lfc.cvar.serrno) + "\n")
return err
# Return the error code
# return rc
return err
def readdirg_grs(*args):
if len(args) < 1:
folder = ''
else:
folder = args[0]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
files = []
guids = []
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder to build :
# - the list of all files with Name, Filemode and GUID,
# - the list of GUID for all regular files.
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if entry.filemode & 040000:
marker = '/'
elif entry.filemode & 020000: # not entry.guid:
marker = '@'
else:
marker = ''
files.append({'name': entry.d_name + marker,
'mode': entry.filemode,
'guid': entry.guid})
if not (entry.filemode & 060000): # Exclude folders and symbolic links
guids.append(entry.guid)
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Get all replicas at once and build dictionaries per GUID
#---------------------------------------------------------------------------
guidSizes = {}
guidReplicas = {}
(res, rep_entries) = lfc.lfc_getreplicas(guids, '')
if res != 0:
print 'lfc_getreplicas : Error ' + str(res) + "\n"
else:
for entry in rep_entries:
if entry.errcode == 0:
if entry.guid in guidReplicas:
guidReplicas[entry.guid].append(entry.sfn)
else:
guidSizes[entry.guid] = entry.filesize
guidReplicas[entry.guid] = [entry.sfn]
#---------------------------------------------------------------------------
# Loop on the list of files to print their Filemode, Name and Replicas
#---------------------------------------------------------------------------
for myfile in files:
print ('%06o' % myfile['mode']) + ' ' + myfile['name']
guid = myfile['guid']
if guid in guidReplicas:
nb_replicas = 0
for replica in guidReplicas[guid]:
if replica != '':
nb_replicas += 1
print ' ==>', replica
print 'Found ' + str(nb_replicas) + ' replica(s) Size=' + \
str(guidSizes[guid])
print
def readdirg_lr(*args):
if len(args) < 1:
folder = ''
else:
folder = args[0]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if entry.filemode & 040000:
marker = '/'
elif entry.filemode & 020000: # not entry.guid:
marker = '@'
else:
marker = ''
print ('%06o' % entry.filemode) + ' ' + entry.d_name + marker
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (entry.filemode & 060000): # Exclude folders and symbolic links
flag = lfc.CNS_LIST_BEGIN
num_replicas = 0
while 1:
res = lfc.lfc_listreplica('', entry.guid, flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
print ' ==>', res.sfn
num_replicas += 1
lfc.lfc_listreplica('', entry.guid, lfc.CNS_LIST_END, listp)
print 'Found ' + str(num_replicas) + ' replica(s)'
print
lfc.lfc_closedir(dir)
# If it is a relative path, prepend $LFC_HOME
#-----------------------------------------------------------------------------
folder = sys.argv[1]
if folder[0] != '/':
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#-----------------------------------------------------------------------------
# Open the folder to get the list of its files
#-----------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
#-----------------------------------------------------------------------------
# Loop on the files contained by the folder.
# If they are neither subfolders nor regular files, we suppose they are
# symbolic links, and we put their name in the 'symlinks' list.
#-----------------------------------------------------------------------------
b_OK = True
symlinks = []
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if (entry.filemode & 040000):
def _getPFNsLFC(guids, lfcHost, storages, nFiles, verbose=False):
# set LFC HOST
os.environ['LFC_HOST'] = lfcHost
# timeout
os.environ['LFC_CONNTIMEOUT'] = '60'
os.environ['LFC_CONRETRY'] = '2'
os.environ['LFC_CONRETRYINT'] = '6'
if verbose:
print "Get file info from %s" % lfcHost
# get PFN
iGUID = 0
nGUID = 10000
pfnMap = {}
listGUID = []
for guid in guids:
iGUID += 1
listGUID.append(guid)
if iGUID % nGUID == 0 or iGUID == len(guids):
sys.stdout.write('.')
sys.stdout.flush()
# get replica
nTry = 3
for iTry in range(nTry):
ret, resList = lfc.lfc_getreplicas(listGUID, '')
if ret != 0 and iTry + 1 < nTry:
print "retry to access %s %s/%s" % (lfcHost, iTry, nTry)
time.sleep(30)
else:
break
if ret == 0:
for fr in resList:
if fr != None and ((not hasattr(fr,'errcode')) or \
(hasattr(fr,'errcode') and fr.errcode == 0)):
# get host
match = re.search('[^:]+://([^:/]+):*\d*/', fr.sfn)
if match == None:
continue
# check host
host = match.group(1)
if storages != [] and (not host in storages):
continue
# skip tape
onTapeFlag = False
for tapePath in [
'/MCTAPE/', '/BNLT1D0/', '/atlasmctape/',
'/atlasdatatape/',
'/castor/cern.ch/grid/atlas/tzero/',
'/castor/cern.ch/grid/atlas/DAQ/'
]:
if re.search(tapePath, fr.sfn) != None:
onTapeFlag = True
break
if onTapeFlag:
continue
# append
if not pfnMap.has_key(fr.guid):
pfnMap[fr.guid] = []
pfnMap[fr.guid].append(fr.sfn)
else:
print "ERROR : %s" % lfc.sstrerror(lfc.cvar.serrno)
sys.exit(EC_LFC)
# reset
listGUID = []
# break
if nFiles > 0 and len(pfnMap) >= nFiles:
break
# return
return pfnMap
