def addMD5sum(self, lfn, md5sum):
""" add md5sum to lfn """
if os.environ.has_key('LD_LIBRARY_PATH'):
tolog("LD_LIBRARY_PATH prior to lfc import: %s" %
os.environ['LD_LIBRARY_PATH'])
else:
tolog(
"!!WARNING!!2999!! LD_LIBRARY_PATH not set prior to lfc import"
)
import lfc
os.environ['LFC_HOST'] = readpar('lfchost')
stat = lfc.lfc_filestatg()
exitcode = lfc.lfc_statg(lfn, "", stat)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("!!WARNING!!2999!! lfc.lfc_statg: %d %s" % (err_num, lfn))
return exitcode
exitcode = lfc.lfc_setfsizeg(stat.guid, stat.filesize, 'MD', md5sum)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("[Non-fatal] ERROR: lfc.lfc_setfsizeg: %d %s %s" %
(err_num, lfn, md5sum))
return exitcode
tolog("Successfully set md5sum for %s" % (lfn))
return exitcode
def addMD5sum(self, lfn, md5sum):
""" add md5sum to lfn """
if os.environ.has_key('LD_LIBRARY_PATH'):
tolog("LD_LIBRARY_PATH prior to lfc import: %s" % os.environ['LD_LIBRARY_PATH'])
else:
tolog("!!WARNING!!2999!! LD_LIBRARY_PATH not set prior to lfc import")
import lfc
os.environ['LFC_HOST'] = readpar('lfchost')
# b="."
# buffer = b.zfill(200)
# ret = lfc.lfc_seterrbuf(buffer, len(buffer))
stat = lfc.lfc_filestatg()
exitcode = lfc.lfc_statg(lfn, "", stat)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("!!WARNING!!2999!! lfc.lfc_statg: %d %s" % (err_num, lfn))
return exitcode
exitcode = lfc.lfc_setfsizeg(stat.guid, stat.filesize, 'MD', md5sum)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("[Non-fatal] ERROR: lfc.lfc_setfsizeg: %d %s %s" % (err_num, lfn, md5sum))
return exitcode
tolog("Successfully set md5sum for %s" % (lfn))
return exitcode
def test(self):
self.name = "/grid/dteam/python_mkdir_test"
self.guid = commands.getoutput('uuidgen').split('/n')[0]
ret = lfc.lfc_mkdirg(self.name, self.guid, 0755)
statg=lfc.lfc_filestatg()
lfc.lfc_statg("",self.guid, statg)
return (statg,ret)
def ret(self):
retval=lfc.lfc_filestatg()
retval.nlink=0
retval.filesize=0L
retval.fileclass=0
retval.status=' '
return retval
def ret(self):
retval=lfc.lfc_filestatg()
retval.nlink=1
retval.filesize=0L
retval.fileclass=0
retval.status=' '
return (retval, errno.EACCES)
def test(self):
self.guid = commands.getoutput('uuidgen').split('/n')[0]
self.name = "/python_filecreatg_test_pd"
ret = lfc.lfc_creatg(self.name,self.guid,0664)
statg=lfc.lfc_filestatg()
ret=lfc.lfc_statg("",self.guid, statg)
return (statg,ret)
def print_GUID(self, path_to_file):
stat = lfc.lfc_filestatg()
res = lfc.lfc_statg(path_to_file, "", stat)
if res == 0:
guid = stat.guid
print path_to_file + ": " + guid
return 0
else:
return self.print_error("There was an error while looking for " + path_to_file)
def test(self):
self.size=987654321L
csumtype=""
csumvalue=""
self.guid="-----------------"
ret=lfc.lfc_setfsizeg(self.guid,self.size,csumtype,csumvalue)
stat=lfc.lfc_filestatg()
lfc.lfc_statg("",self.guid, stat)
return (stat,ret)
def print_GUID(path_to_file):
#name = "/grid/belle/MC/test/testua/hoge1.root"
stat = lfc.lfc_filestatg()
res = lfc.lfc_statg(path_to_file, "", stat)
if res == 0:
guid = stat.guid
print path_to_file + ": " + guid
return 0
else:
return print_error("There was an error while looking for " + path_to_file)
def ret(self):
retval=lfc.lfc_filestatg()
retval.nlink=0
retval.filesize=0L
retval.atime=1184059742
retval.mtime=1171381061
retval.ctime=1171381061
retval.fileclass=0
retval.status=' '
return retval
def ret(self):
retval=lfc.lfc_filestatg()
retval.fileid=0L
retval.filemode=0
retval.nlink=-1
retval.uid=0
retval.gid=0
retval.filesize=0L
retval.atime=0
retval.mtime=0
retval.ctime=0
retval.fileclass=0
retval.status=' '
retval.guid=""
retval.csumtype=""
retval.csumvalue=""
return retval
def ret(self):
retval=lfc.lfc_filestatg()
retval.fileid=25159L
retval.filemode=33204
retval.nlink=1
retval.uid=137
retval.gid=101
retval.filesize=0L
retval.atime=1184059742
retval.mtime=1171381061
retval.ctime=1171381061
retval.fileclass=0
retval.status='-'
retval.guid=self.guid
retval.csumtype="w"
retval.csumvalue=" "
return retval
def checkDir(nickname):
first_letter = nickname[0]
dirname = '/grid/lhcb/user/'
dirname += first_letter
dirname += '/'
dirname += nickname
stat = lfc.lfc_filestatg()
res = lfc.lfc_statg(dirname, "", stat)
if res == 0:
print "Directory exists "
return 0
else:
err_num = lfc.cvar.serrno #@UndefinedVariable
err_string = lfc.sstrerror(err_num)
print "There was an error while looking for " + dirname + ": Error " + str(
err_num) + " (" + err_string + ")"
if err_num == 2:
res = createDir(dirname, nickname)
return res
else:
return -3
def addMD5sum(self, lfn, md5sum):
""" add md5sum to lfn """
if os.environ.has_key("LD_LIBRARY_PATH"):
tolog("LD_LIBRARY_PATH prior to lfc import: %s" % os.environ["LD_LIBRARY_PATH"])
else:
tolog("!!WARNING!!2999!! LD_LIBRARY_PATH not set prior to lfc import")
import lfc
os.environ["LFC_HOST"] = readpar("lfchost")
stat = lfc.lfc_filestatg()
exitcode = lfc.lfc_statg(lfn, "", stat)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("!!WARNING!!2999!! lfc.lfc_statg: %d %s" % (err_num, lfn))
return exitcode
exitcode = lfc.lfc_setfsizeg(stat.guid, stat.filesize, "MD", md5sum)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("[Non-fatal] ERROR: lfc.lfc_setfsizeg: %d %s %s" % (err_num, lfn, md5sum))
return exitcode
tolog("Successfully set md5sum for %s" % (lfn))
return exitcode
def ret(self):
retval=lfc.lfc_filestatg()
retval.status='D'
return retval
def test(self):
ret = lfc.lfc_delete(name)
statg=lfc.lfc_filestatg()
lfc.lfc_statg(name,"", statg)
return (statg,ret)
def test(self):
ret = lfc.lfc_unlink(self.name)
err = lfc.cvar.serrno
statg=lfc.lfc_filestatg()
lfc.lfc_statg(self.name,"", statg)
return ((statg, err), ret)
def test(self):
ret = lfc.lfc_unlink(self.name)
statg=lfc.lfc_filestatg()
lfc.lfc_statg(self.name,"", statg)
return (statg,ret)
print columns
print columns_sub
try:
while replica is not None:
res = str(replica.fileid)
if options.surl:
res += "|" + str(replica.sfn).replace("\0", "")
if options.lfn:
lfc.lfc_getpath("", replica.fileid, lfn)
res = res + "|" + lfn.strip().replace("\0", "")
if options.owner:
buf = lfc.lfc_filestatg()
lfc.lfc_statr(replica.sfn, buf)
username = "******" * 1024
if buf.uid not in users:
if lfc.lfc_getusrbyuid(buf.uid, username) == 0:
# Adding user to the list in order to resolve him/her only once
dn = username.strip().replace("\0", "")
users[buf.uid] = dn
else:
users[buf.uid] = "unknown"
res = res + "|" + users[buf.uid]
print res
flags = lfc.CNS_LIST_CONTINUE
def ret(self):
statg=lfc.lfc_filestatg()
statg.guid = ""
return statg
#
# test entry
#
directory = "/grid/dteam/my_test_dir/"
name = "/grid/dteam/my_test_dir/my.test"
replica1 = "sfn://my_se.in2p3.fr/hpss/in2p3.fr/group/sophie/tests_python/dir/my.test"
replica2 = "srm://my_other_se.cern.ch/castor/cern.ch/grid/sophie/tests_python/dir/my.test"
replica3 = "sfn://my_se.in2p3.fr/hpss/in2p3.fr/group/sophie/tests_python/dir/my.test2"
status = '-'
f_type = 'D'
#
# delete test entry, if exists
#
stat1 = lfc.lfc_filestatg()
if (lfc.lfc_statg(name,"",stat1)) == 0:
if (lfc.lfc_delreplica(stat1.guid, None, replica1)) != 0:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
print "error"+ str(err_num) + " (" + err_string + ")"
lfc.lfc_delreplica(stat1.guid, None, replica2)
lfc.lfc_delreplica(stat1.guid, None, replica3)
lfc.lfc_unlink(name)
lfc.lfc_rmdir(directory)
#
# create entry in LFC for following tests
#
guid = commands.getoutput('uuidgen').split('\n')[0]
def test(self):
statg=lfc.lfc_filestatg()
ret=lfc.lfc_statr(self.sfn, statg)
return (statg,ret)
def test(self):
ret = lfc.lfc_undelete(os.path.dirname(name))
err = lfc.cvar.serrno
statg=lfc.lfc_filestatg()
lfc.lfc_statg(name,"", statg)
return ((statg, err), ret)
def test(self):
ret = lfc.lfc_delete(name)
err = lfc.cvar.serrno
statg=lfc.lfc_filestatg()
lfc.lfc_statg(name,"", statg)
return ((statg, err), ret)
class dCacheLFCSiteMover(dCacheSiteMover.dCacheSiteMover):
"""
SiteMover for dCache/dccp with LFC based replica catalog
"""
copyCommand = "dccplfc"
checksum_command = "adler32"
has_mkdir = True
has_df = False
has_getsize = True
has_md5sum = False
has_chmod = True
timeout = 5 * 3600
def __init__(self, setup_path='', *args, **kwrds):
self._setup = setup_path
def get_timeout(self):
return self.timeout
def _check_space(self, ub):
"""dCache specific space verification.
There is no way at the moment to verify full dCache space availability"""
return 999999
def addMD5sum(self, lfn, md5sum):
""" add md5sum to lfn """
if os.environ.has_key('LD_LIBRARY_PATH'):
tolog("LD_LIBRARY_PATH prior to lfc import: %s" %
os.environ['LD_LIBRARY_PATH'])
else:
tolog(
"!!WARNING!!2999!! LD_LIBRARY_PATH not set prior to lfc import"
)
try:
import lfc
except Exception, e:
tolog(
"!!WARNING!!2999!! addMD5Sum() could not import lfc module: %s"
% str(e))
return -1
os.environ['LFC_HOST'] = readpar('lfchost')
# b="."
# buffer = b.zfill(200)
# ret = lfc.lfc_seterrbuf(buffer, len(buffer))
stat = lfc.lfc_filestatg()
exitcode = lfc.lfc_statg(lfn, "", stat)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("!!WARNING!!2999!! lfc.lfc_statg: %d %s" % (err_num, lfn))
return exitcode
exitcode = lfc.lfc_setfsizeg(stat.guid, stat.filesize, 'MD', md5sum)
if exitcode != 0:
# print "error:",buffer
err_num = lfc.cvar.serrno
tolog("[Non-fatal] ERROR: lfc.lfc_setfsizeg: %d %s %s" %
(err_num, lfn, md5sum))
return exitcode
tolog("Successfully set md5sum for %s" % (lfn))
return exitcode
def test(self):
self.new_name="/grid/dteam/python_filerenamed_test"
ret = lfc.lfc_rename(self.name,self.new_name)
stat=lfc.lfc_filestatg()
ret=lfc.lfc_statg(self.new_name, "", stat)
return (stat,ret)
def prepare(self):
self.guid = commands.getoutput('uuidgen').split('/n')[0]
self.name = "/grid/dteam/python_filerename_test"
ret = lfc.lfc_creatg(self.name,self.guid,0664)
self.stat=lfc.lfc_filestatg()
ret=lfc.lfc_statg(self.name, "", self.stat)
def list_replicas(*names):
#---------------------------------------------------------------------------
# Loop on the file names given as parameters
#---------------------------------------------------------------------------
for name in names:
if name[0] != '/':
if 'LFC_HOME' in os.environ:
name = os.environ['LFC_HOME'] + '/' + name
else:
sys.exit('Relative folder path requires LFC_HOME to be set and exported')
#-------------------------------------------------------------------------
# stat an existing entry in the LFC and print the GUID
#-------------------------------------------------------------------------
statg = lfc.lfc_filestatg()
res = lfc.lfc_statg(name, '', statg)
if res != 0:
err_num = lfc.cvar.serrno
err_string = lfc.sstrerror(err_num)
sys.exit('Error ' + str(err_num) + ' while looking for ' + name + ': ' + err_string)
if statg.filemode & 040000:
print ('%06o' % statg.filemode), name, 'is a folder'
guid = statg.guid
if guid:
print ('%06o' % statg.filemode), name, 'has guid:' + guid
else:
print ('%06o' % statg.filemode), name, 'has NO guid'
#-------------------------------------------------------------------------
# retrieve the comment on a file
#-------------------------------------------------------------------------
buffer = ' ' * lfc.CA_MAXCOMMENTLEN
res = lfc.lfc_getcomment(name, buffer)
if res != 0:
err_num = lfc.cvar.serrno
if err_num != 2:
err_string = lfc.sstrerror(err_num)
print 'Error ' + str(err_num) + ' while reading the comment for ' + name + ': ' + err_string
else:
print "Comment: '" + buffer.rstrip(' ') + "'"
#-------------------------------------------------------------------------
# list the replicas of a given entry, starting from the GUID
#-------------------------------------------------------------------------
listp = lfc.lfc_list()
flag = lfc.CNS_LIST_BEGIN
num_replicas = 0
while 1:
res = lfc.lfc_listreplica('', guid, flag, listp)
if res == None:
break
else:
flag = lfc.CNS_LIST_CONTINUE
print ' ==>', res.sfn
num_replicas += 1
lfc.lfc_listreplica('', guid, lfc.CNS_LIST_END, listp)
print 'Found ' + str(num_replicas) + ' replica(s)\n'
print columns
print columns_sub
try:
while replica is not None:
res = str(replica.fileid)
if options.surl:
res += "|" + str(replica.sfn).replace('\0', '')
if options.lfn:
lfc.lfc_getpath("", replica.fileid, lfn)
res = res + "|" + lfn.strip().replace('\0', '')
if options.owner:
buf = lfc.lfc_filestatg()
lfc.lfc_statr(replica.sfn, buf)
username = "******" * 1024
if buf.uid not in users:
if lfc.lfc_getusrbyuid(buf.uid, username) == 0:
# Adding user to the list in order to resolve him/her only once
dn = username.strip().replace('\0', '')
users[buf.uid] = dn
else:
users[buf.uid] = "unknown"
res = res + "|" + users[buf.uid]
print res
flags = lfc.CNS_LIST_CONTINUE
farray = [file,]
retCode = 0
# Remove if it was already there
lfc.lfc_delreplicasbysfn([file.sfn,], [""])
# Register
print "Registering file"
if lfc.lfc_registerfiles(farray)[0] != 0:
print "Failed to register the file: %s" % lfc.sstrerror(lfc.cvar.serrno)
sys.exit(1)
# Do the lfc_statr and check
statg = lfc.lfc_filestatg()
if lfc.lfc_statr(file.sfn, statg) != 0:
print "Failed to do the lfc_statr: %s" % lfc.sstrerror(lfc.cvar.serrno)
retCode = 1
if statg.csumtype != file.csumtype or statg.csumvalue != file.csumvalue or statg.guid != file.guid:
print "Cns_statr repbuffer too small!"
retCode = 1
else:
print "Test for lfc_statr passed!!"
# Try with statg
if lfc.lfc_statg("", file.guid, statg) != 0:
print "Failed to do the lfc_statg: %s" % lfc.sstrerror(lfc.cvar.serrno)
retCode = 1
def test(self):
ret = lfc.lfc_creatg(self.name,self.guid,0664)
statg=lfc.lfc_filestatg()
ret=lfc.lfc_statg("",self.guid, statg)
return (statg,ret)
def test(self):
statg=lfc.lfc_filestatg()
ret=lfc.lfc_statr("sfn://test-se.cern.ch/grid/dteam/hary/lfc_statr_test1", statg)
return (statg,ret)