Ejemplo n.º 1
0
    def getFiles(self, run=None, dataset=None, lumi=None, verbose=0):
        """
        Return list of lfns for given run/(dataset,lumi)
        """
        params = dict(self.params)
        params.update({"detail":True})
        if  run:
            params.update({'run':run})
# switch to run_num when DBS3 is ready
#            params.update({'run_num':run})
        if  dataset:
            params.update({'dataset':dataset})
        if  lumi:
            params.update({'lumi_list':lumi})
        query  = "find file,file.size where"
        fileList = []
        for dbs in self.dbslist:
            try:
                if  verbose:
                    print dbs
                dbsurl = self.getdbsurl(dbs) + '/files?%s' \
                    % urllib.urlencode(params, doseq=True)
                data   = urllib2.urlopen(dbsurl)
                files  = json.load(data)
                for row in files:
                    lfn = row['logical_file_name']
                    size = row['file_size']
                    fileList.append([lfn, sizeFormat(size)])
                if  fileList:
                    print "Found files in %s" % dbs
                    return fileList
            except:
                pass
        return fileList
Ejemplo n.º 2
0
def parseDBSoutput(data):
    """
    Parse DBS XML output
    """
    dom = parseString(data)
    oList = []
    for node in dom.getElementsByTagName('result'):
        if node.hasAttribute('FILES_FILESIZE') and \
                node.hasAttribute('FILES_LOGICALFILENAME'):
            oList.append((str(node.getAttribute('FILES_LOGICALFILENAME')),
                sizeFormat(node.getAttribute('FILES_FILESIZE')) ))
        elif node.hasAttribute('FILES_LOGICALFILENAME') and \
                node.hasAtribute('APPVERSION_VERSION'):
            oList.append((str(node.getAttribute('FILES_LOGICALFILENAME')),
                node.getAttribute('APPVERSION_VERSION') ))
        elif node.hasAttribute('FILES_LOGICALFILENAME'):
            oList.append(str(node.getAttribute('FILES_LOGICALFILENAME')))
        elif node.hasAttribute('DATATIER_NAME'):
            oList.append(node.getAttribute('DATATIER_NAME'))
        elif node.hasAttribute('BLOCK_NAME') and \
                node.hasAttribute('STORAGEELEMENT_SENAME'):
            oList.append((node.getAttribute('BLOCK_NAME'),
                node.getAttribute('STORAGEELEMENT_SENAME')))
        elif node.hasAttribute('BLOCK_NAME'):
            oList.append(node.getAttribute('BLOCK_NAME'))
        elif node.hasAttribute('APPVERSION_VERSION'):
            oList.append(node.getAttribute('APPVERSION_VERSION'))
    return oList
Ejemplo n.º 3
0
 def checkUserCache(self, user):
     """check users's cache"""
     page = ""
     lfnList, statList = self.userDict[user]
     self.makedir(user)
     pfnList = os.listdir("%s/%s/softlinks" % (self.download_dir, user))
     for ifile in pfnList:
         f = "%s/%s/softlinks/%s" % (self.download_dir, user, ifile)
         abspath = os.readlink(f)
         if  not os.path.isfile(abspath):
             # we got orphan link
             try:
                 os.remove(f)
             except Exception as _exc:
                 pass
             continue
         fileStat = os.stat(abspath)
         fileSize = sizeFormat(fileStat[stat.ST_SIZE])
         link     = "download/%s/%s" % (user, ifile)
         lfn      = abspath.replace(self.transfer_dir, "")
         msg  = "<a href=\"%s/%s\">Download (%s)</a> " \
                     % (self.url, link, fileSize)
         if  not lfnList.count(lfn):
             lfnList.append("%s" % lfn)
             status = (StatusCode.DONE, msg)
             statList.append(status)
     self.userDict[user] = (lfnList, statList)
     page += self.updateUserPage(user)
     return page
Ejemplo n.º 4
0
 def updatePageWithLfnInfo(self, user, lfn):
     """Update page with LFN info"""
     page  = ""
     lfnList, statList = self.userDict[user]
     self.makedir(user)
     if  not lfnList:
         return ""
     try:
         idx = lfnList.index(lfn)
         statusCode, statusMsg  = statList[idx]
         if  statusCode == StatusCode.DONE:
             filename = lfn.split('/')[-1]
             pfn      = os.path.join(self.transfer_dir, lfn[1:])
             if  os.path.isfile(pfn):
                 if  not os.path.isfile("%s/%s/%s" \
                     % (self.download_dir, user, filename)):
                     try:
                         os.link(pfn, "%s/%s/%s" \
                             % (self.download_dir, user, filename))
                         os.symlink(pfn, "%s/%s/softlinks/%s" \
                             % (self.download_dir, user, filename))
                     except Exception as exc:
                         print_exc(exc)
                 link     = "download/%s/%s" % (user, filename)
                 filepath = "%s/%s/%s" % (self.download_dir, user, filename)
                 fileStat = os.stat(filepath)
                 fileSize = sizeFormat(fileStat[stat.ST_SIZE])
                 msg  = "<a href=\"%s/%s\">Download (%s)</a>" \
                             % (self.url, link, fileSize)
                 statList[idx] = (StatusCode.DONE, msg)
             else:
                 statList[idx] = (StatusCode.TRANSFER_STATUS_UNKNOWN, \
                     StatusMsg.TRANSFER_STATUS_UNKNOWN)
                 msg = cgi.escape(StatusMsg.TRANSFER_STATUS_UNKNOWN)
             page += msg + " | " + removeLfn(lfn)
         else:
             page += cgi.escape(statusMsg) + " | " + cancelLfn(lfn)
     except ValueError as err:
         print_exc(err)
         print lfn
         print self.userDict
     except Exception as exc:
         print_exc(exc)
         print lfn
         print self.userDict
     return page
Ejemplo n.º 5
0
def parseDBSoutput_DBS_2_0_6(data):
    """
    DBS XML parser for DBS server DBS_2_0_6 and later
    """
    dom  = parseString(data)
    datalist = []
    for node in dom.getElementsByTagName('row'):
        olist = []
        for child in node.childNodes:
            subnode = child.firstChild
            if  not subnode:
                continue
            if  child.nodeName == 'file.size':
                data = sizeFormat(subnode.data)
            else:
                data = subnode.data
            olist.append(data)
        if  len(olist) == 1:
            datalist.append(olist[-1])
        else:
            datalist.append(olist)
    return datalist