def readdirg_recurse(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
folderFiles = {prefix: []}
#---------------------------------------------------------------------------
# Append the entries of the folder to the current list
#---------------------------------------------------------------------------
files = []
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
files.append({'name': entry.d_name,
'mode': entry.filemode})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the entries of the current file list to build the dictionary
#---------------------------------------------------------------------------
for myfile in files:
folderFiles[prefix].append(myfile['name'])
#-------------------------------------------------------------------------
# If the entry is a folder, recurse
#-------------------------------------------------------------------------
if myfile['mode'] & 040000:
folderFiles.update(readdirg_recurse(prefix + myfile['name']))
return folderFiles
def readdirg_recurse(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
files = []
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder to build the list of files
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
files.append({'name': prefix + entry.d_name,
'mode': entry.filemode,
'guid': entry.guid})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the current file list : If the entry is a folder, recurse
#---------------------------------------------------------------------------
for myfile in files[:]: # Local copy is mandatory
if myfile['mode'] & 040000:
files.extend(readdirg_recurse(myfile['name']))
return files
if __name__ == "__main__":
parser = OptionParser(usage="%prog <se_name> [--surl] [--dn] [--lfn]", version="%prog 1.3")
parser.add_option("--surl", action="store_true", dest="surl", default=False, help="Get Storage URL")
parser.add_option("--dn", action="store_true", dest="owner", default=False, help="Get Owner")
parser.add_option("--lfn", action="store_true", dest="lfn", default=False, help="Get LFN")
(options, args) = parser.parse_args()
if len(args) >= 3 or len(args) < 1:
parser.error("wrong number of parameters")
se_name = args[0]
users = {}
flags = lfc.CNS_LIST_BEGIN
lista = lfc.lfc_list()
replica = lfc.lfc_listreplicax("", se_name, "", flags, lista)
lfn = "\0" * 4096
columns = "fileId"
columns_sub = "------"
if options.surl:
columns += " | SURL"
columns_sub += "-----------"
if options.lfn:
columns += " | LFN"
columns_sub += "----------"
if options.owner:
columns += " | Owner's DN"
columns_sub += "-----------------"
def list_replicas(*names):
#---------------------------------------------------------------------------
# Loop on the file names given as parameters
#---------------------------------------------------------------------------
for name in names:
if name[0] != '/':
if 'LFC_HOME' in os.environ:
name = os.environ['LFC_HOME'] + '/' + name
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#-------------------------------------------------------------------------
# stat an existing entry in the LFC and print the GUID
#-------------------------------------------------------------------------
statg = lfc.lfc_filestatg()
res = lfc.lfc_statg(name, '', statg)
if res != 0:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' while looking for ' + name + ': ' + err_string)
if statg.filemode & 040000:
print ('%06o' % statg.filemode), name, 'is a folder'
guid = statg.guid
if guid:
print ('%06o' % statg.filemode), name, 'has guid:' + guid
else:
print ('%06o' % statg.filemode), name, 'has NO guid'
#-------------------------------------------------------------------------
# retrieve the comment on a file
#-------------------------------------------------------------------------
buffer = ' ' * lfc.CA_MAXCOMMENTLEN
res = lfc.lfc_getcomment(name, buffer)
if res != 0:
err_num = lfc.cvar.serrno
if err_num != 2:
err_string = lfc.sstrerror(err_num)
print 'Error ' + str(err_num) + ' while reading the comment for ' + name + ': ' + err_string
else:
print "Comment: '" + buffer.rstrip(' ') + "'"
#-------------------------------------------------------------------------
# list the replicas of a given entry, starting from the GUID
#-------------------------------------------------------------------------
listp = lfc.lfc_list()
flag = lfc.CNS_LIST_BEGIN
num_replicas = 0
while 1:
res = lfc.lfc_listreplica('', guid, flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
print ' ==>', res.sfn
num_replicas += 1
lfc.lfc_listreplica('', guid, lfc.CNS_LIST_END, listp)
print 'Found ' + str(num_replicas) + ' replica(s)\n'
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
print "[ERROR] There was an error while looking for " + name + ": Error " + str(err_num) + " (" + err_string + ")"
#
# add replicas
#
lfc.lfc_addreplica(guid, None, os.getenv("LFC_HOST"), replica1, status, f_type, "", "")
lfc.lfc_addreplica(guid, None, os.getenv("LFC_HOST"), replica2, status, f_type, "", "")
lfc.lfc_addreplica(guid, None, os.getenv("LFC_HOST"), replica3, status, f_type, "", "")
#
# list replicas, starting from the GUID
#
listp = lfc.lfc_list()
flag = lfc.CNS_LIST_BEGIN
print "Listing replicas for GUID " + guid
num_replicas=0
while(1):
res = lfc.lfc_listreplica("",guid,flag,listp)
flag = lfc.CNS_LIST_CONTINUE
if res == None:
break
else:
rep_name = res.sfn
print "Replica: " + rep_name
def readdirg_lr(*args):
if len(args) < 1:
folder = ''
else:
folder = args[0]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if entry.filemode & 040000:
marker = '/'
elif entry.filemode & 020000: # not entry.guid:
marker = '@'
else:
marker = ''
print ('%06o' % entry.filemode) + ' ' + entry.d_name + marker
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (entry.filemode & 060000): # Exclude folders and symbolic links
flag = lfc.CNS_LIST_BEGIN
num_replicas = 0
while 1:
res = lfc.lfc_listreplica('', entry.guid, flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
print ' ==>', res.sfn
num_replicas += 1
lfc.lfc_listreplica('', entry.guid, lfc.CNS_LIST_END, listp)
print 'Found ' + str(num_replicas) + ' replica(s)'
print
lfc.lfc_closedir(dir)
def readdirg_lr_recurse_timestamp(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
time_format = '%Y/%m/%d %H:%M:%S %Z'
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
name = prefix + entry.d_name
sys.stdout.write(time.strftime(time_format) + ' ' + name + "\n")
sys.stdout.flush()
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (entry.filemode & 060000): # Exclude folders and symbolic links
flag = lfc.CNS_LIST_BEGIN
while 1:
res = lfc.lfc_listreplica('', entry.guid, flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
sys.stdout.write(time.strftime(time_format) + ' ==> ' + res.sfn + "\n")
sys.stdout.flush()
lfc.lfc_listreplica('', entry.guid, lfc.CNS_LIST_END, listp)
#-------------------------------------------------------------------------
# If the entry is a folder, recurse
#-------------------------------------------------------------------------
if entry.filemode & 040000:
readdirg_lr_recurse_timestamp(name)
lfc.lfc_closedir(dir)
def deleteOneEntry(pLfn, pDir, pKeepLfn, pRemoveLinks, pVerbose):
"""
Tries to delete all the replicas associated with the specified LFN (or
symlink). After all the replicas have been removed and if pKeepLfn is not
true, then the LFN or symlink given is also removed.
If pRemoveLinks == True, then all the other links (including the main
LFN) are also removed.
pKeepLfn and pRemoveLinks should not be true at the same time. In such a
case, results are uncertain.
If the specified LFN is relative, then pDir should contain its parent
directory, so that links can be checked to be in the same directory (in
order to use relative names, what should give faster operation). If pDir
is "", or if the lfn is absolute (for whatever value of pDir), absolute
names are used for the LFN and sym links deletion.
The specified LFN is used as it is, without using the LFC_HOME env
variable (that should be pre-pended by the caller, or the LFC directory
changed to it beforehand in order to use the relative path).
"""
lfn = pLfn
if (lfn.startswith('/')):
dir = ""
else:
dir = pDir
keepLfn = pKeepLfn
removeLinks = pRemoveLinks
verbose = pVerbose
if (verbose):
print "--Deleting entry: " + lfn
rc = 0
# Delete the replicas (unconditional)
listrep = lfc.lfc_list()
flagsrep = lfc.CNS_LIST_BEGIN
# Call the retrieval in a loop
rc = 0
filerep = lfc.lfc_listreplica(lfn, "", flagsrep, listrep)
while (filerep):
if (verbose):
print "--lfc.lfc_listreplica(\"" + lfn + "\", \"\",", flagsrep, ",listrep)"
err = deleteOneReplica(filerep.sfn, verbose)
if (err): rc = err
flagsrep = lfc.CNS_LIST_CONTINUE
filerep = lfc.lfc_listreplica(lfn, "", flagsrep, listrep)
flagsrep = lfc.CNS_LIST_END
lfc.lfc_listreplica(lfn, "", flagsrep, listrep)
# Now go on with LFN and sym links: if -l was specified, delete them all
if (removeLinks):
list = lfc.lfc_list()
flags = lfc.CNS_LIST_BEGIN
link = lfc.lfc_listlinks(lfn, "", flags, list)
while (link):
if (verbose):
print "--lfc.lfc_listlinks(\"" + lfn + "\", \"\",", flags, ",list)"
linkpath = link.path
# For links in the same directory, we use the relative name
if (dir):
linkdir = linkpath[0:linkpath.rfind('/')]
if (linkdir == dir):
linkpath = linkpath[linkpath.rfind('/') + 1:]
err = deleteOneLFN(linkpath, verbose)
if (err): rc = err
flags = lfc.CNS_LIST_CONTINUE
link = lfc.lfc_listlinks(lfn, "", flags, list)
flags = lfc.CNS_LIST_END
lfc.lfc_listlinks(lfn, "", flags, list)
# If not done yet, remove also the given LFN or link (unless -k was specified)
if ((not removeLinks) and (not keepLfn)):
err = deleteOneLFN(lfn, verbose)
if (err): rc = err
# Return the error code
return rc
def readdirg_gr(*args):
if len(args) < 1:
folder = ''
else:
folder = args[0]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
files = []
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder to build the list of files
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if entry.filemode & 040000:
marker = '/'
elif entry.filemode & 020000: # not entry.guid:
marker = '@'
else:
marker = ''
files.append({'name': entry.d_name + marker,
'mode': entry.filemode,
'guid': entry.guid})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the list of files
#---------------------------------------------------------------------------
lfc.lfc_startsess('', '')
for myfile in files:
print ('%06o' % myfile['mode']) + ' ' + myfile['name']
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (myfile['mode'] & 060000): # Exclude folders and symbolic links
(res, replicas) = lfc.lfc_getreplica('', myfile['guid'], '')
if res == 0:
for replica in replicas:
print ' ==>', replica.sfn
print 'Found ' + str(len(replicas)) + ' replica(s)'
print
lfc.lfc_endsess()
def deleteOneEntry(pLfn, pDir, pKeepLfn, pRemoveLinks, pVerbose):
"""
Tries to delete all the replicas associated with the specified LFN (or
symlink). After all the replicas have been removed and if pKeepLfn is not
true, then the LFN or symlink given is also removed.
If pRemoveLinks == True, then all the other links (including the main
LFN) are also removed.
pKeepLfn and pRemoveLinks should not be true at the same time. In such a
case, results are uncertain.
If the specified LFN is relative, then pDir should contain its parent
directory, so that links can be checked to be in the same directory (in
order to use relative names, what should give faster operation). If pDir
is "", or if the lfn is absolute (for whatever value of pDir), absolute
names are used for the LFN and sym links deletion.
The specified LFN is used as it is, without using the LFC_HOME env
variable (that should be pre-pended by the caller, or the LFC directory
changed to it beforehand in order to use the relative path).
"""
lfn = pLfn
if(lfn.startswith('/')):
dir = ""
else:
dir = pDir
keepLfn = pKeepLfn
removeLinks = pRemoveLinks
verbose = pVerbose
if(verbose):
print "--Deleting entry: "+lfn
rc = 0
# Delete the replicas (unconditional)
listrep=lfc.lfc_list()
flagsrep=lfc.CNS_LIST_BEGIN
# Call the retrieval in a loop
rc=0
filerep=lfc.lfc_listreplica(lfn, "", flagsrep, listrep)
while(filerep):
if(verbose):
print "--lfc.lfc_listreplica(\""+lfn+"\", \"\",",flagsrep,",listrep)"
err = deleteOneReplica(filerep.sfn, verbose)
if(err): rc=err
flagsrep=lfc.CNS_LIST_CONTINUE
filerep=lfc.lfc_listreplica(lfn, "", flagsrep, listrep)
flagsrep=lfc.CNS_LIST_END
lfc.lfc_listreplica(lfn, "", flagsrep, listrep)
# Now go on with LFN and sym links: if -l was specified, delete them all
if(removeLinks):
list=lfc.lfc_list()
flags=lfc.CNS_LIST_BEGIN
link=lfc.lfc_listlinks(lfn, "", flags, list)
while(link):
if(verbose):
print "--lfc.lfc_listlinks(\""+lfn+"\", \"\",",flags,",list)"
linkpath = link.path
# For links in the same directory, we use the relative name
if(dir):
linkdir = linkpath[0:linkpath.rfind('/')]
if(linkdir == dir):
linkpath = linkpath[linkpath.rfind('/')+1:]
err = deleteOneLFN(linkpath, verbose)
if(err): rc=err
flags=lfc.CNS_LIST_CONTINUE
link=lfc.lfc_listlinks(lfn, "", flags, list)
flags=lfc.CNS_LIST_END
lfc.lfc_listlinks(lfn, "", flags, list)
# If not done yet, remove also the given LFN or link (unless -k was specified)
if((not removeLinks) and (not keepLfn)):
err = deleteOneLFN(lfn, verbose)
if(err): rc = err
# Return the error code
return rc
help="Get Owner")
parser.add_option("--lfn",
action="store_true",
dest="lfn",
default=False,
help="Get LFN")
(options, args) = parser.parse_args()
if len(args) >= 3 or len(args) < 1:
parser.error("wrong number of parameters")
se_name = args[0]
users = {}
flags = lfc.CNS_LIST_BEGIN
lista = lfc.lfc_list()
replica = lfc.lfc_listreplicax("", se_name, "", flags, lista)
lfn = "\0" * 4096
columns = "fileId"
columns_sub = "------"
if options.surl:
columns += " | SURL"
columns_sub += "-----------"
if options.lfn:
columns += " | LFN"
columns_sub += "----------"
if options.owner:
columns += " | Owner's DN"
columns_sub += "-----------------"
def readdirg_lr_recurse(*args):
if len(args) < 1:
folder = ''
prefix = ''
else:
folder = args[0]
prefix = folder + '/'
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + prefix
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Append the entries of the folder to the current list
#---------------------------------------------------------------------------
files = []
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
files.append({'name': prefix + entry.d_name,
'mode': entry.filemode,
'guid': entry.guid})
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Loop on the entries of the current file list
#---------------------------------------------------------------------------
for myfile in files:
print myfile['name']
#-------------------------------------------------------------------------
# If the entry is a regular file, list its replicas using its GUID
#-------------------------------------------------------------------------
if not (myfile['mode'] & 060000): # Exclude folders and symbolic links
flag = lfc.CNS_LIST_BEGIN
while 1:
res = lfc.lfc_listreplica('', myfile['guid'], flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
print ' ==>', res.sfn
lfc.lfc_listreplica('', myfile['guid'], lfc.CNS_LIST_END, listp)
#-------------------------------------------------------------------------
# If the entry is a folder, recurse
#-------------------------------------------------------------------------
if myfile['mode'] & 040000:
readdirg_lr_recurse(myfile['name'])
def readdirg_grs(*args):
if len(args) < 1:
folder = ''
else:
folder = args[0]
if (folder == '') or (folder[0] != '/'):
if 'LFC_HOME' in os.environ:
folder = os.environ['LFC_HOME'] + '/' + folder
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#---------------------------------------------------------------------------
# Open the folder
#---------------------------------------------------------------------------
dir = lfc.lfc_opendirg(folder, '')
if dir == None:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string)
files = []
guids = []
listp = lfc.lfc_list()
#---------------------------------------------------------------------------
# Loop on the entries of the folder to build :
# - the list of all files with Name, Filemode and GUID,
# - the list of GUID for all regular files.
#---------------------------------------------------------------------------
while 1:
entry = lfc.lfc_readdirg(dir)
if entry == None:
break
if entry.filemode & 040000:
marker = '/'
elif entry.filemode & 020000: # not entry.guid:
marker = '@'
else:
marker = ''
files.append({'name': entry.d_name + marker,
'mode': entry.filemode,
'guid': entry.guid})
if not (entry.filemode & 060000): # Exclude folders and symbolic links
guids.append(entry.guid)
lfc.lfc_closedir(dir)
#---------------------------------------------------------------------------
# Get all replicas at once and build dictionaries per GUID
#---------------------------------------------------------------------------
guidSizes = {}
guidReplicas = {}
(res, rep_entries) = lfc.lfc_getreplicas(guids, '')
if res != 0:
print 'lfc_getreplicas : Error ' + str(res) + "\n"
else:
for entry in rep_entries:
if entry.errcode == 0:
if entry.guid in guidReplicas:
guidReplicas[entry.guid].append(entry.sfn)
else:
guidSizes[entry.guid] = entry.filesize
guidReplicas[entry.guid] = [entry.sfn]
#---------------------------------------------------------------------------
# Loop on the list of files to print their Filemode, Name and Replicas
#---------------------------------------------------------------------------
for myfile in files:
print ('%06o' % myfile['mode']) + ' ' + myfile['name']
guid = myfile['guid']
if guid in guidReplicas:
nb_replicas = 0
for replica in guidReplicas[guid]:
if replica != '':
nb_replicas += 1
print ' ==>', replica
print 'Found ' + str(nb_replicas) + ' replica(s) Size=' + \
str(guidSizes[guid])
print
