def deleteEntries(pLfnList, pKeepLfn, pRemoveLinks, pVerbose): """ For each LFN in pLfnList, the contents of LFC_HOME env var are prepended for the relative paths (absoulte paths are used as they are), and deleteOneEntry is invoked. Please check deleteOneEntry for the meaning of the other arguments. """ lfnList = pLfnList keepLfn = pKeepLfn removeLinks = pRemoveLinks verbose = pVerbose rc = 0 for lfn in lfnList: lfn = lfn.strip() [lfn, dir] = checkHomeDir(lfn) err = lfc.lfc_chdir(dir) if (err < 0): sys.stderr.write("Error changing dir to: " + dir + " : " + lfc.sstrerror(lfc.cvar.serrno) + "\n") return err err = deleteOneEntry(lfn, dir, keepLfn, removeLinks, verbose) if (err): rc = err # Return the error code return rc
def deleteEntries(pLfnList, pKeepLfn, pRemoveLinks, pVerbose): """ For each LFN in pLfnList, the contents of LFC_HOME env var are prepended for the relative paths (absoulte paths are used as they are), and deleteOneEntry is invoked. Please check deleteOneEntry for the meaning of the other arguments. """ lfnList = pLfnList keepLfn = pKeepLfn removeLinks = pRemoveLinks verbose = pVerbose rc = 0 for lfn in lfnList: lfn = lfn.strip() [lfn, dir] = checkHomeDir(lfn) err = lfc.lfc_chdir(dir) if(err < 0): sys.stderr.write("Error changing dir to: "+dir+" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n") return err err = deleteOneEntry(lfn, dir, keepLfn, removeLinks, verbose) if(err): rc=err # Return the error code return rc
def readdirxr_recurse(*args): if len(args) < 1: folder = '' prefix = '' else: folder = args[0] prefix = folder + '/' if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + prefix else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) #--------------------------------------------------------------------------- # Loop on the entries of the folder #--------------------------------------------------------------------------- folders = [] while 1: read_pt = lfc.lfc_readdirxr(dir, '') if read_pt == None: break entry, replicas = read_pt name = prefix + entry.d_name if entry.filemode & 040000: folders.append(name) comment = ' (Folder)' elif entry.nbreplicas == 0: comment = ' (No replica)' else: comment = '' print name + comment #------------------------------------------------------------------------- # If the entry is a regular file, list its replicas #------------------------------------------------------------------------- if comment == '': for replica in replicas: print ' ==>', replica.sfn lfc.lfc_closedir(dir) #--------------------------------------------------------------------------- # Recurse on subfolders #--------------------------------------------------------------------------- for name in folders: readdirxr_recurse(name)
def readdirxr(folderFiles): folderKeys = folderFiles.keys() folderKeys.sort() #print folderKeys #--------------------------------------------------------------------------- # Loop on the folders #--------------------------------------------------------------------------- for folder in folderKeys: prefix = folder #print '\n' + prefix + ':', folderFiles[prefix] if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + folder else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #------------------------------------------------------------------------- # Open the folder #------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) #------------------------------------------------------------------------- # Loop on the entries of the folder which are in the dictionary #------------------------------------------------------------------------- while 1: read_pt = lfc.lfc_readdirxr(dir, '') if read_pt == None: break entry, replicas = read_pt if entry.d_name not in folderFiles[prefix]: break if entry.filemode & 040000: comment = ' (Folder)' elif entry.nbreplicas == 0: comment = ' (No replica)' else: comment = '' print prefix + entry.d_name #----------------------------------------------------------------------- # If the entry is a regular file, list its replicas #----------------------------------------------------------------------- if comment == '': for replica in replicas: print ' ==>', replica.sfn lfc.lfc_closedir(dir)
def delUser(userDN, _userid): res = lfc.lfc_rmusrmap(0, userDN) if res == 0: print "User has been removed from userinfo" return 0 else: err_num = lfc.cvar.serrno #@UndefinedVariable err_string = lfc.sstrerror(err_num) print "There was an error while deleting user: Error " + str( err_num) + " (" + err_string + ")" return -1
def readdirg_recurse(*args): if len(args) < 1: folder = '' prefix = '' else: folder = args[0] prefix = folder + '/' if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + prefix else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) listp = lfc.lfc_list() folderFiles = {prefix: []} #--------------------------------------------------------------------------- # Append the entries of the folder to the current list #--------------------------------------------------------------------------- files = [] while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break files.append({'name': entry.d_name, 'mode': entry.filemode}) lfc.lfc_closedir(dir) #--------------------------------------------------------------------------- # Loop on the entries of the current file list to build the dictionary #--------------------------------------------------------------------------- for myfile in files: folderFiles[prefix].append(myfile['name']) #------------------------------------------------------------------------- # If the entry is a folder, recurse #------------------------------------------------------------------------- if myfile['mode'] & 040000: folderFiles.update(readdirg_recurse(prefix + myfile['name'])) return folderFiles
def rmDir(nickname): dirname = "/grid/lhcb/user/" + nickname[0] + "/" + nickname res = lfc.lfc_rmdir(dirname) if res == 0: print "user dir has been removed " return 0 else: err_num = lfc.cvar.serrno #@UndefinedVariable err_string = lfc.sstrerror(err_num) print "There was an error while deleting dir: Error " + str( err_num) + " (" + err_string + ")" return -1
def deleteOneReplica(pSfn, pVerbose): """ Tries to delete the replica information with the SFN specified. """ verbose = pVerbose if(verbose): print "--lfc.lfc_delreplica(\"\", None, \""+pSfn+"\")" if(lfc.lfc_delreplica("", None, pSfn) < 0): sys.stderr.write("Warning: Error deleting replica:"+pSfn+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n") return (-1)
def updateDirPerm(userid, nickname, userDN): dirname = "/grid/lhcb/user/" + nickname[0] + "/" + nickname print "updating owner of user dir to " + str(userid) res = lfc.lfc_chown(dirname, userid, -1) if res == 0: print "Owner directory updated" return 0 else: err_num = lfc.cvar.serrno #@UndefinedVariable err_string = lfc.sstrerror(err_num) print "There was an error while updating ownerid for " + dirname + " Error " + str( err_num) + " (" + err_string + ")" return -1
def deleteOneReplica(pSfn, pVerbose): """ Tries to delete the replica information with the SFN specified. """ verbose = pVerbose if (verbose): print "--lfc.lfc_delreplica(\"\", None, \"" + pSfn + "\")" if (lfc.lfc_delreplica("", None, pSfn) < 0): sys.stderr.write("Warning: Error deleting replica:" + pSfn + ": " + lfc.sstrerror(lfc.cvar.serrno) + "\n") return (-1)
def deleteOneLFN(pLfn, pVerbose): """ Tries to delete the LFN (or sym link) specified """ lfn = pLfn verbose = pVerbose if (verbose): print "--lfc.lfc_unlink(\"" + lfn + "\")" if (lfc.lfc_unlink(lfn) < 0): sys.stderr.write("Warning: Error removing LFN:" + lfn + ": " + lfc.sstrerror(lfc.cvar.serrno) + "\n") return -1
def deleteOneLFN(pLfn, pVerbose): """ Tries to delete the LFN (or sym link) specified """ lfn=pLfn verbose = pVerbose if(verbose): print "--lfc.lfc_unlink(\""+lfn+"\")" if(lfc.lfc_unlink(lfn)<0): sys.stderr.write("Warning: Error removing LFN:"+lfn+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n") return -1
def createDir(dirname, _nickname): res = lfc.lfc_mkdir(dirname, 0777) if res == 0: print "Directory " + dirname + " has been created and " + str(res) #we delete the directory #res = lfc.lfc_rmdir(dirname) return -1 else: err_num = lfc.cvar.serrno #@UndefinedVariable err_string = lfc.sstrerror(err_num) print "There was an error while creating " + dirname + " Error " + str( err_num) + " (" + err_string + ")" return -2
def getUserName(userID): result, u_list = lfc.lfc_getusrmap() if (result == 0): for i in u_list: if i.userid == userID: print "userID exists mapped to " + str(i.username) return i.username print "Did not find the corresponding username for " + str(userID) return "ERROR" else: err_num = lfc.cvar.serrno #@UndefinedVariable err_string = lfc.sstrerror(err_num) print "There was an error while looking at username for " + str( userID) + " Error " + str(err_num) + " (" + err_string + ")" return "ERROR"
def readdirg_recurse(*args): if len(args) < 1: folder = '' prefix = '' else: folder = args[0] prefix = folder + '/' if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + prefix else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) files = [] listp = lfc.lfc_list() #--------------------------------------------------------------------------- # Loop on the entries of the folder to build the list of files #--------------------------------------------------------------------------- while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break files.append({'name': prefix + entry.d_name, 'mode': entry.filemode, 'guid': entry.guid}) lfc.lfc_closedir(dir) #--------------------------------------------------------------------------- # Loop on the current file list : If the entry is a folder, recurse #--------------------------------------------------------------------------- for myfile in files[:]: # Local copy is mandatory if myfile['mode'] & 040000: files.extend(readdirg_recurse(myfile['name'])) return files
def createUser(userDN, userid): res = lfc.lfc_enterusrmap(userid, userDN) if res == 0: print "User created " + userDN return 0 else: err_num = lfc.cvar.serrno #@UndefinedVariable err_string = lfc.sstrerror(err_num) # error not due to userDN exists if err_num != 17: print "There was an error while adding " + userDN + " Error " + str( err_num) + " (" + err_string + ")" return -1 #however we return 0 because we want to update the dir as it has been created by root else: print "UserDN already exists" return 0
def checkDir(nickname): first_letter = nickname[0] dirname = '/grid/lhcb/user/' dirname += first_letter dirname += '/' dirname += nickname stat = lfc.lfc_filestatg() res = lfc.lfc_statg(dirname, "", stat) if res == 0: print "Directory exists " return 0 else: err_num = lfc.cvar.serrno #@UndefinedVariable err_string = lfc.sstrerror(err_num) print "There was an error while looking for " + dirname + ": Error " + str( err_num) + " (" + err_string + ")" if err_num == 2: res = createDir(dirname, nickname) return res else: return -3
def readdirg(*args): if len(args) < 1: folder = '' else: folder = args[0] if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + folder else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) #--------------------------------------------------------------------------- # Loop on the entries of the folder #--------------------------------------------------------------------------- while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break if entry.filemode & 040000: marker = '/' elif entry.filemode & 020000: # not entry.guid: marker = '@' else: marker = '' print ('%06o' % entry.filemode) + ' ' + entry.d_name + marker lfc.lfc_closedir(dir)
def del_replicas(sfns,guids): rem_guids=[] rem_sfns=[] for sfn_prefix in SFN_PREFIXES: prefixed_sfns=[ sfn_prefix + sfn for sfn in sfns ] result,err_nums=lfc.lfc_delreplicasbysfn(prefixed_sfns,guids) #print err_nums for i in range(len(err_nums)): err_num=err_nums[i] if err_num == 2: rem_guids.append(guids[i]) rem_sfns.append(sfns[i]) elif err_num != 0: err_string = lfc.sstrerror(err_num) print "There was an error for guid " + guids[i] + ": " + str(err_num) + " (" + err_string + ")" guids=rem_guids sfns=rem_sfns rem_guids=[] rem_sfns=[] if len(guids) < 0: break for guid in guids: print "There was an error for guid " + guid + ": 2 (No such file or directory), Correct SFN may not be in list or file already deleted."
def deleteOneDir(pDir, pForce, pExtLinks, pExtLfns, pVerbose): """ Tries to delete the specified directory. If "pForce" is true, then the directory will be removed even if it is not empty and all the entries in the directory (including subdirectories) will be removed as well, in a recursive way. Otherwise, the deletion will be performed only if the directory is empty. If pExtLinks == pExtLfns == False, the function will remove only the specified directory contents and will not to touch external sym links pointing to LFNs in the directory, nor entries pointed to by links located in the directory. If pExtLinks == True, then links located in other directories that point to LFNs in the specified directory are also removed. If pExtLfns == True, then LFNs (and their replicas and all their sym links) located in other directories and pointed to by links in the specified directory are also removed. """ dir = pDir force = pForce extLinks = pExtLinks extLfns = pExtLfns verbose = pVerbose rc = 0 err = 0 subdirlist = [] # Subdirectories to remove when we are done with current one [dir, parentdir] = lfc_del_lfn.checkHomeDir(dir) fulldir = parentdir+dir if(verbose): print "--Deleting Dir: "+dir err = lfc.lfc_chdir(fulldir) if(err < 0): sys.stderr.write("Error changing dir to: "+fulldir+" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n") return err dir_p=lfc.lfc_DIR() dir_entry=lfc.lfc_direnstatg() dir_p=lfc.lfc_opendirg(".", "") if(dir_p < 0): sys.stderr.write("Error opening specified dir: "+dir+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n") return -1 # Read first entry lfc.lfc_rewinddir(dir_p) dir_entry=lfc.lfc_readdirg(dir_p) if(force): # Remove all LFNs in a loop S_IFDIR = 0x4000 S_IFLNK = 0xA000 while(dir_entry): lfn = dir_entry.d_name if(dir_entry.filemode & S_IFDIR): # This entry is a directory subdirlist.append(lfn) else: if((dir_entry.filemode & S_IFLNK) == S_IFLNK): # This entry is a sym link if(extLfns): # Remove the replicas and all the links (including main LFN) err = lfc_del_lfn.deleteOneEntry(lfn, fulldir, False, True, verbose) else: # Remove only the sym link (no replicas) err = lfc_del_lfn.deleteOneLFN(lfn, verbose) else: # This entry is a main LFN # # First check if the file has been alredy deleted (links, etc...) # fstat = lfc.lfc_filestatg() # if(lfc.lfc_statg(lfn, "", fstat)<0): # if(verbose): # print "--Warning. Skipping deletion of non-accessible file:",lfn,":",\ # lfc.sstrerror(lfc.cvar.serrno) # else: if(extLinks): # Remove the replicas and all the links that point to this LFN err = lfc_del_lfn.deleteOneEntry(lfn, fulldir, False, True, verbose) else: # Remove only this LFN and replicas (but no external sym links) err = lfc_del_lfn.deleteOneEntry(lfn, fulldir, False, False, verbose) if(err): rc = err dir_entry=lfc.lfc_readdirg(dir_p) else: if(dir_entry): sys.stderr.write("Error: Directory "+dir+" not empty! Consider using -r.\n") return -1 # Close the directory if(lfc.lfc_closedir(dir_p) < 0): sys.stderr.write("Error closing dir: "+dir+" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n") # Remove all subdirectories in the list for subdir in subdirlist: err = deleteOneDir(fulldir+'/'+subdir, force, extLinks, extLfns, verbose) if(err): rc=err # Finally, remove also the top directory itself err = lfc.lfc_chdir("..") if(err < 0): sys.stderr.write("Error changing dir to \"..\" : "+lfc.sstrerror(lfc.cvar.serrno)+"\n") return err if(verbose): print "--lfc.lfc_unlink(\""+dir+"\")" err = lfc.lfc_rmdir(dir) if(err<0): sys.stderr.write("Error removing dir: "+dir+": "+lfc.sstrerror(lfc.cvar.serrno)+"\n") return err # Return the error code # return rc return err
def readdirg_lr_recurse_timestamp(*args): if len(args) < 1: folder = '' prefix = '' else: folder = args[0] prefix = folder + '/' if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + prefix else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') time_format = '%Y/%m/%d %H:%M:%S %Z' #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) listp = lfc.lfc_list() #--------------------------------------------------------------------------- # Loop on the entries of the folder #--------------------------------------------------------------------------- while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break name = prefix + entry.d_name sys.stdout.write(time.strftime(time_format) + ' ' + name + "\n") sys.stdout.flush() #------------------------------------------------------------------------- # If the entry is a regular file, list its replicas using its GUID #------------------------------------------------------------------------- if not (entry.filemode & 060000): # Exclude folders and symbolic links flag = lfc.CNS_LIST_BEGIN while 1: res = lfc.lfc_listreplica('', entry.guid, flag, listp) if res == None: break else: flag = lfc.CNS_LIST_CONTINUE sys.stdout.write(time.strftime(time_format) + ' ==> ' + res.sfn + "\n") sys.stdout.flush() lfc.lfc_listreplica('', entry.guid, lfc.CNS_LIST_END, listp) #------------------------------------------------------------------------- # If the entry is a folder, recurse #------------------------------------------------------------------------- if entry.filemode & 040000: readdirg_lr_recurse_timestamp(name) lfc.lfc_closedir(dir)
file.sfn = "srm://example.com/path/" file.server = "example.com" file.csumtype = "MD" file.csumvalue = "1a31009319c99ebdefd23055b20ff034" farray = [file,] retCode = 0 # Remove if it was already there lfc.lfc_delreplicasbysfn([file.sfn,], [""]) # Register print "Registering file" if lfc.lfc_registerfiles(farray)[0] != 0: print "Failed to register the file: %s" % lfc.sstrerror(lfc.cvar.serrno) sys.exit(1) # Do the lfc_statr and check statg = lfc.lfc_filestatg() if lfc.lfc_statr(file.sfn, statg) != 0: print "Failed to do the lfc_statr: %s" % lfc.sstrerror(lfc.cvar.serrno) retCode = 1 if statg.csumtype != file.csumtype or statg.csumvalue != file.csumvalue or statg.guid != file.guid: print "Cns_statr repbuffer too small!" retCode = 1 else: print "Test for lfc_statr passed!!" # Try with statg
def _getPFNsLFC(guids,lfcHost,storages,nFiles,verbose=False): # set LFC HOST os.environ['LFC_HOST'] = lfcHost # timeout os.environ['LFC_CONNTIMEOUT'] = '60' os.environ['LFC_CONRETRY'] = '2' os.environ['LFC_CONRETRYINT'] = '6' if verbose: print "Get file info from %s" % lfcHost # get PFN iGUID = 0 nGUID = 10000 pfnMap = {} listGUID = [] for guid in guids: iGUID += 1 listGUID.append(guid) if iGUID % nGUID == 0 or iGUID == len(guids): sys.stdout.write('.') sys.stdout.flush() # get replica nTry = 3 for iTry in range(nTry): ret,resList = lfc.lfc_getreplicas(listGUID,'') if ret != 0 and iTry+1<nTry: print "retry to access %s %s/%s" % (lfcHost,iTry,nTry) time.sleep(30) else: break if ret == 0: for fr in resList: if fr != None and ((not hasattr(fr,'errcode')) or \ (hasattr(fr,'errcode') and fr.errcode == 0)): # get host match = re.search('[^:]+://([^:/]+):*\d*/',fr.sfn) if match==None: continue # check host host = match.group(1) if storages != [] and (not host in storages): continue # skip tape onTapeFlag = False for tapePath in ['/MCTAPE/','/BNLT1D0/','/atlasmctape/','/atlasdatatape/', '/castor/cern.ch/grid/atlas/tzero/', '/castor/cern.ch/grid/atlas/DAQ/']: if re.search(tapePath,fr.sfn) != None: onTapeFlag = True break if onTapeFlag: continue # append if not pfnMap.has_key(fr.guid): pfnMap[fr.guid] = [] pfnMap[fr.guid].append(fr.sfn) else: print "ERROR : %s" % lfc.sstrerror(lfc.cvar.serrno) sys.exit(EC_LFC) # reset listGUID = [] # break if nFiles > 0 and len(pfnMap) >= nFiles: break # return return pfnMap
def print_error(self, msg): err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) print >> sys.stderr, msg, "Error " + str(err_num) + " (" + err_string + ")" return err_num
def list_replicas(*names): #--------------------------------------------------------------------------- # Loop on the file names given as parameters #--------------------------------------------------------------------------- for name in names: if name[0] != '/': if 'LFC_HOME' in os.environ: name = os.environ['LFC_HOME'] + '/' + name else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #------------------------------------------------------------------------- # stat an existing entry in the LFC and print the GUID #------------------------------------------------------------------------- statg = lfc.lfc_filestatg() res = lfc.lfc_statg(name, '', statg) if res != 0: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' while looking for ' + name + ': ' + err_string) if statg.filemode & 040000: print ('%06o' % statg.filemode), name, 'is a folder' guid = statg.guid if guid: print ('%06o' % statg.filemode), name, 'has guid:' + guid else: print ('%06o' % statg.filemode), name, 'has NO guid' #------------------------------------------------------------------------- # retrieve the comment on a file #------------------------------------------------------------------------- buffer = ' ' * lfc.CA_MAXCOMMENTLEN res = lfc.lfc_getcomment(name, buffer) if res != 0: err_num = lfc.cvar.serrno if err_num != 2: err_string = lfc.sstrerror(err_num) print 'Error ' + str(err_num) + ' while reading the comment for ' + name + ': ' + err_string else: print "Comment: '" + buffer.rstrip(' ') + "'" #------------------------------------------------------------------------- # list the replicas of a given entry, starting from the GUID #------------------------------------------------------------------------- listp = lfc.lfc_list() flag = lfc.CNS_LIST_BEGIN num_replicas = 0 while 1: res = lfc.lfc_listreplica('', guid, flag, listp) if res == None: break else: flag = lfc.CNS_LIST_CONTINUE print ' ==>', res.sfn num_replicas += 1 lfc.lfc_listreplica('', guid, lfc.CNS_LIST_END, listp) print 'Found ' + str(num_replicas) + ' replica(s)\n'
def readdirg_lr_recurse(*args): if len(args) < 1: folder = '' prefix = '' else: folder = args[0] prefix = folder + '/' if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + prefix else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) listp = lfc.lfc_list() #--------------------------------------------------------------------------- # Append the entries of the folder to the current list #--------------------------------------------------------------------------- files = [] while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break files.append({'name': prefix + entry.d_name, 'mode': entry.filemode, 'guid': entry.guid}) lfc.lfc_closedir(dir) #--------------------------------------------------------------------------- # Loop on the entries of the current file list #--------------------------------------------------------------------------- for myfile in files: print myfile['name'] #------------------------------------------------------------------------- # If the entry is a regular file, list its replicas using its GUID #------------------------------------------------------------------------- if not (myfile['mode'] & 060000): # Exclude folders and symbolic links flag = lfc.CNS_LIST_BEGIN while 1: res = lfc.lfc_listreplica('', myfile['guid'], flag, listp) if res == None: break else: flag = lfc.CNS_LIST_CONTINUE print ' ==>', res.sfn lfc.lfc_listreplica('', myfile['guid'], lfc.CNS_LIST_END, listp) #------------------------------------------------------------------------- # If the entry is a folder, recurse #------------------------------------------------------------------------- if myfile['mode'] & 040000: readdirg_lr_recurse(myfile['name'])
def readdirg_gr(*args): if len(args) < 1: folder = '' else: folder = args[0] if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + folder else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) files = [] listp = lfc.lfc_list() #--------------------------------------------------------------------------- # Loop on the entries of the folder to build the list of files #--------------------------------------------------------------------------- while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break if entry.filemode & 040000: marker = '/' elif entry.filemode & 020000: # not entry.guid: marker = '@' else: marker = '' files.append({'name': entry.d_name + marker, 'mode': entry.filemode, 'guid': entry.guid}) lfc.lfc_closedir(dir) #--------------------------------------------------------------------------- # Loop on the list of files #--------------------------------------------------------------------------- lfc.lfc_startsess('', '') for myfile in files: print ('%06o' % myfile['mode']) + ' ' + myfile['name'] #------------------------------------------------------------------------- # If the entry is a regular file, list its replicas using its GUID #------------------------------------------------------------------------- if not (myfile['mode'] & 060000): # Exclude folders and symbolic links (res, replicas) = lfc.lfc_getreplica('', myfile['guid'], '') if res == 0: for replica in replicas: print ' ==>', replica.sfn print 'Found ' + str(len(replicas)) + ' replica(s)' print lfc.lfc_endsess()
def deleteOneDir(pDir, pForce, pExtLinks, pExtLfns, pVerbose): """ Tries to delete the specified directory. If "pForce" is true, then the directory will be removed even if it is not empty and all the entries in the directory (including subdirectories) will be removed as well, in a recursive way. Otherwise, the deletion will be performed only if the directory is empty. If pExtLinks == pExtLfns == False, the function will remove only the specified directory contents and will not to touch external sym links pointing to LFNs in the directory, nor entries pointed to by links located in the directory. If pExtLinks == True, then links located in other directories that point to LFNs in the specified directory are also removed. If pExtLfns == True, then LFNs (and their replicas and all their sym links) located in other directories and pointed to by links in the specified directory are also removed. """ dir = pDir force = pForce extLinks = pExtLinks extLfns = pExtLfns verbose = pVerbose rc = 0 err = 0 subdirlist = [ ] # Subdirectories to remove when we are done with current one [dir, parentdir] = lfc_del_lfn.checkHomeDir(dir) fulldir = parentdir + dir if (verbose): print "--Deleting Dir: " + dir err = lfc.lfc_chdir(fulldir) if (err < 0): sys.stderr.write("Error changing dir to: " + fulldir + " : " + lfc.sstrerror(lfc.cvar.serrno) + "\n") return err dir_p = lfc.lfc_DIR() dir_entry = lfc.lfc_direnstatg() dir_p = lfc.lfc_opendirg(".", "") if (dir_p < 0): sys.stderr.write("Error opening specified dir: " + dir + ": " + lfc.sstrerror(lfc.cvar.serrno) + "\n") return -1 # Read first entry lfc.lfc_rewinddir(dir_p) dir_entry = lfc.lfc_readdirg(dir_p) if (force): # Remove all LFNs in a loop S_IFDIR = 0x4000 S_IFLNK = 0xA000 while (dir_entry): lfn = dir_entry.d_name if (dir_entry.filemode & S_IFDIR): # This entry is a directory subdirlist.append(lfn) else: if ((dir_entry.filemode & S_IFLNK) == S_IFLNK): # This entry is a sym link if (extLfns): # Remove the replicas and all the links (including main LFN) err = lfc_del_lfn.deleteOneEntry( lfn, fulldir, False, True, verbose) else: # Remove only the sym link (no replicas) err = lfc_del_lfn.deleteOneLFN(lfn, verbose) else: # This entry is a main LFN # # First check if the file has been alredy deleted (links, etc...) # fstat = lfc.lfc_filestatg() # if(lfc.lfc_statg(lfn, "", fstat)<0): # if(verbose): # print "--Warning. Skipping deletion of non-accessible file:",lfn,":",\ # lfc.sstrerror(lfc.cvar.serrno) # else: if (extLinks): # Remove the replicas and all the links that point to this LFN err = lfc_del_lfn.deleteOneEntry( lfn, fulldir, False, True, verbose) else: # Remove only this LFN and replicas (but no external sym links) err = lfc_del_lfn.deleteOneEntry( lfn, fulldir, False, False, verbose) if (err): rc = err dir_entry = lfc.lfc_readdirg(dir_p) else: if (dir_entry): sys.stderr.write("Error: Directory " + dir + " not empty! Consider using -r.\n") return -1 # Close the directory if (lfc.lfc_closedir(dir_p) < 0): sys.stderr.write("Error closing dir: " + dir + " : " + lfc.sstrerror(lfc.cvar.serrno) + "\n") # Remove all subdirectories in the list for subdir in subdirlist: err = deleteOneDir(fulldir + '/' + subdir, force, extLinks, extLfns, verbose) if (err): rc = err # Finally, remove also the top directory itself err = lfc.lfc_chdir("..") if (err < 0): sys.stderr.write("Error changing dir to \"..\" : " + lfc.sstrerror(lfc.cvar.serrno) + "\n") return err if (verbose): print "--lfc.lfc_unlink(\"" + dir + "\")" err = lfc.lfc_rmdir(dir) if (err < 0): sys.stderr.write("Error removing dir: " + dir + ": " + lfc.sstrerror(lfc.cvar.serrno) + "\n") return err # Return the error code # return rc return err
def readdirg_grs(*args): if len(args) < 1: folder = '' else: folder = args[0] if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + folder else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) files = [] guids = [] listp = lfc.lfc_list() #--------------------------------------------------------------------------- # Loop on the entries of the folder to build : # - the list of all files with Name, Filemode and GUID, # - the list of GUID for all regular files. #--------------------------------------------------------------------------- while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break if entry.filemode & 040000: marker = '/' elif entry.filemode & 020000: # not entry.guid: marker = '@' else: marker = '' files.append({'name': entry.d_name + marker, 'mode': entry.filemode, 'guid': entry.guid}) if not (entry.filemode & 060000): # Exclude folders and symbolic links guids.append(entry.guid) lfc.lfc_closedir(dir) #--------------------------------------------------------------------------- # Get all replicas at once and build dictionaries per GUID #--------------------------------------------------------------------------- guidSizes = {} guidReplicas = {} (res, rep_entries) = lfc.lfc_getreplicas(guids, '') if res != 0: print 'lfc_getreplicas : Error ' + str(res) + "\n" else: for entry in rep_entries: if entry.errcode == 0: if entry.guid in guidReplicas: guidReplicas[entry.guid].append(entry.sfn) else: guidSizes[entry.guid] = entry.filesize guidReplicas[entry.guid] = [entry.sfn] #--------------------------------------------------------------------------- # Loop on the list of files to print their Filemode, Name and Replicas #--------------------------------------------------------------------------- for myfile in files: print ('%06o' % myfile['mode']) + ' ' + myfile['name'] guid = myfile['guid'] if guid in guidReplicas: nb_replicas = 0 for replica in guidReplicas[guid]: if replica != '': nb_replicas += 1 print ' ==>', replica print 'Found ' + str(nb_replicas) + ' replica(s) Size=' + \ str(guidSizes[guid]) print
def readdirg_lr(*args): if len(args) < 1: folder = '' else: folder = args[0] if (folder == '') or (folder[0] != '/'): if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + folder else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #--------------------------------------------------------------------------- # Open the folder #--------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) listp = lfc.lfc_list() #--------------------------------------------------------------------------- # Loop on the entries of the folder #--------------------------------------------------------------------------- while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break if entry.filemode & 040000: marker = '/' elif entry.filemode & 020000: # not entry.guid: marker = '@' else: marker = '' print ('%06o' % entry.filemode) + ' ' + entry.d_name + marker #------------------------------------------------------------------------- # If the entry is a regular file, list its replicas using its GUID #------------------------------------------------------------------------- if not (entry.filemode & 060000): # Exclude folders and symbolic links flag = lfc.CNS_LIST_BEGIN num_replicas = 0 while 1: res = lfc.lfc_listreplica('', entry.guid, flag, listp) if res == None: break else: flag = lfc.CNS_LIST_CONTINUE print ' ==>', res.sfn num_replicas += 1 lfc.lfc_listreplica('', entry.guid, lfc.CNS_LIST_END, listp) print 'Found ' + str(num_replicas) + ' replica(s)' print lfc.lfc_closedir(dir)
# If it is a relative path, prepend $LFC_HOME #----------------------------------------------------------------------------- folder = sys.argv[1] if folder[0] != '/': if 'LFC_HOME' in os.environ: folder = os.environ['LFC_HOME'] + '/' + folder else: sys.exit('Relative folder path requires LFC_HOME to be set and exported') #----------------------------------------------------------------------------- # Open the folder to get the list of its files #----------------------------------------------------------------------------- dir = lfc.lfc_opendirg(folder, '') if dir == None: err_num = lfc.cvar.serrno err_string = lfc.sstrerror(err_num) sys.exit('Error ' + str(err_num) + ' on folder ' + folder + ': ' + err_string) #----------------------------------------------------------------------------- # Loop on the files contained by the folder. # If they are neither subfolders nor regular files, we suppose they are # symbolic links, and we put their name in the 'symlinks' list. #----------------------------------------------------------------------------- b_OK = True symlinks = [] while 1: entry = lfc.lfc_readdirg(dir) if entry == None: break if (entry.filemode & 040000):
def _getPFNsLFC(guids, lfcHost, storages, nFiles, verbose=False): # set LFC HOST os.environ['LFC_HOST'] = lfcHost # timeout os.environ['LFC_CONNTIMEOUT'] = '60' os.environ['LFC_CONRETRY'] = '2' os.environ['LFC_CONRETRYINT'] = '6' if verbose: print "Get file info from %s" % lfcHost # get PFN iGUID = 0 nGUID = 10000 pfnMap = {} listGUID = [] for guid in guids: iGUID += 1 listGUID.append(guid) if iGUID % nGUID == 0 or iGUID == len(guids): sys.stdout.write('.') sys.stdout.flush() # get replica nTry = 3 for iTry in range(nTry): ret, resList = lfc.lfc_getreplicas(listGUID, '') if ret != 0 and iTry + 1 < nTry: print "retry to access %s %s/%s" % (lfcHost, iTry, nTry) time.sleep(30) else: break if ret == 0: for fr in resList: if fr != None and ((not hasattr(fr,'errcode')) or \ (hasattr(fr,'errcode') and fr.errcode == 0)): # get host match = re.search('[^:]+://([^:/]+):*\d*/', fr.sfn) if match == None: continue # check host host = match.group(1) if storages != [] and (not host in storages): continue # skip tape onTapeFlag = False for tapePath in [ '/MCTAPE/', '/BNLT1D0/', '/atlasmctape/', '/atlasdatatape/', '/castor/cern.ch/grid/atlas/tzero/', '/castor/cern.ch/grid/atlas/DAQ/' ]: if re.search(tapePath, fr.sfn) != None: onTapeFlag = True break if onTapeFlag: continue # append if not pfnMap.has_key(fr.guid): pfnMap[fr.guid] = [] pfnMap[fr.guid].append(fr.sfn) else: print "ERROR : %s" % lfc.sstrerror(lfc.cvar.serrno) sys.exit(EC_LFC) # reset listGUID = [] # break if nFiles > 0 and len(pfnMap) >= nFiles: break # return return pfnMap