def getFiles(self, run=None, dataset=None, lumi=None, verbose=0):
"""
Return list of lfns for given run/(dataset,lumi)
"""
params = dict(self.params)
params.update({"detail":True})
if run:
params.update({'run':run})
# switch to run_num when DBS3 is ready
# params.update({'run_num':run})
if dataset:
params.update({'dataset':dataset})
if lumi:
params.update({'lumi_list':lumi})
query = "find file,file.size where"
fileList = []
for dbs in self.dbslist:
try:
if verbose:
print dbs
dbsurl = self.getdbsurl(dbs) + '/files?%s' \
% urllib.urlencode(params, doseq=True)
data = urllib2.urlopen(dbsurl)
files = json.load(data)
for row in files:
lfn = row['logical_file_name']
size = row['file_size']
fileList.append([lfn, sizeFormat(size)])
if fileList:
print "Found files in %s" % dbs
return fileList
except:
pass
return fileList
def parseDBSoutput(data):
"""
Parse DBS XML output
"""
dom = parseString(data)
oList = []
for node in dom.getElementsByTagName('result'):
if node.hasAttribute('FILES_FILESIZE') and \
node.hasAttribute('FILES_LOGICALFILENAME'):
oList.append((str(node.getAttribute('FILES_LOGICALFILENAME')),
sizeFormat(node.getAttribute('FILES_FILESIZE')) ))
elif node.hasAttribute('FILES_LOGICALFILENAME') and \
node.hasAtribute('APPVERSION_VERSION'):
oList.append((str(node.getAttribute('FILES_LOGICALFILENAME')),
node.getAttribute('APPVERSION_VERSION') ))
elif node.hasAttribute('FILES_LOGICALFILENAME'):
oList.append(str(node.getAttribute('FILES_LOGICALFILENAME')))
elif node.hasAttribute('DATATIER_NAME'):
oList.append(node.getAttribute('DATATIER_NAME'))
elif node.hasAttribute('BLOCK_NAME') and \
node.hasAttribute('STORAGEELEMENT_SENAME'):
oList.append((node.getAttribute('BLOCK_NAME'),
node.getAttribute('STORAGEELEMENT_SENAME')))
elif node.hasAttribute('BLOCK_NAME'):
oList.append(node.getAttribute('BLOCK_NAME'))
elif node.hasAttribute('APPVERSION_VERSION'):
oList.append(node.getAttribute('APPVERSION_VERSION'))
return oList
def checkUserCache(self, user):
"""check users's cache"""
page = ""
lfnList, statList = self.userDict[user]
self.makedir(user)
pfnList = os.listdir("%s/%s/softlinks" % (self.download_dir, user))
for ifile in pfnList:
f = "%s/%s/softlinks/%s" % (self.download_dir, user, ifile)
abspath = os.readlink(f)
if not os.path.isfile(abspath):
# we got orphan link
try:
os.remove(f)
except Exception as _exc:
pass
continue
fileStat = os.stat(abspath)
fileSize = sizeFormat(fileStat[stat.ST_SIZE])
link = "download/%s/%s" % (user, ifile)
lfn = abspath.replace(self.transfer_dir, "")
msg = "<a href=\"%s/%s\">Download (%s)</a> " \
% (self.url, link, fileSize)
if not lfnList.count(lfn):
lfnList.append("%s" % lfn)
status = (StatusCode.DONE, msg)
statList.append(status)
self.userDict[user] = (lfnList, statList)
page += self.updateUserPage(user)
return page
def updatePageWithLfnInfo(self, user, lfn):
"""Update page with LFN info"""
page = ""
lfnList, statList = self.userDict[user]
self.makedir(user)
if not lfnList:
return ""
try:
idx = lfnList.index(lfn)
statusCode, statusMsg = statList[idx]
if statusCode == StatusCode.DONE:
filename = lfn.split('/')[-1]
pfn = os.path.join(self.transfer_dir, lfn[1:])
if os.path.isfile(pfn):
if not os.path.isfile("%s/%s/%s" \
% (self.download_dir, user, filename)):
try:
os.link(pfn, "%s/%s/%s" \
% (self.download_dir, user, filename))
os.symlink(pfn, "%s/%s/softlinks/%s" \
% (self.download_dir, user, filename))
except Exception as exc:
print_exc(exc)
link = "download/%s/%s" % (user, filename)
filepath = "%s/%s/%s" % (self.download_dir, user, filename)
fileStat = os.stat(filepath)
fileSize = sizeFormat(fileStat[stat.ST_SIZE])
msg = "<a href=\"%s/%s\">Download (%s)</a>" \
% (self.url, link, fileSize)
statList[idx] = (StatusCode.DONE, msg)
else:
statList[idx] = (StatusCode.TRANSFER_STATUS_UNKNOWN, \
StatusMsg.TRANSFER_STATUS_UNKNOWN)
msg = cgi.escape(StatusMsg.TRANSFER_STATUS_UNKNOWN)
page += msg + " | " + removeLfn(lfn)
else:
page += cgi.escape(statusMsg) + " | " + cancelLfn(lfn)
except ValueError as err:
print_exc(err)
print lfn
print self.userDict
except Exception as exc:
print_exc(exc)
print lfn
print self.userDict
return page
def parseDBSoutput_DBS_2_0_6(data):
"""
DBS XML parser for DBS server DBS_2_0_6 and later
"""
dom = parseString(data)
datalist = []
for node in dom.getElementsByTagName('row'):
olist = []
for child in node.childNodes:
subnode = child.firstChild
if not subnode:
continue
if child.nodeName == 'file.size':
data = sizeFormat(subnode.data)
else:
data = subnode.data
olist.append(data)
if len(olist) == 1:
datalist.append(olist[-1])
else:
datalist.append(olist)
return datalist