Example #1
0
def getDBDatasets(jobO,trf,dbrelease):
    from pandatools import Client

    # get DB datasets
    dbrFiles  = {}
    dbrDsList = []
    if trf or dbrelease != '':
        if trf:
            # parse jobO for TRF
            tmpItems = jobO.split()
        else:
            # mimic a trf parameter to reuse following algorithm
            tmpItems = ['%DB='+dbrelease]
        # look for DBRelease
        for tmpItem in tmpItems:
            match = re.search('%DB=([^:]+):(.+)$',tmpItem)
            if match:
                tmpDbrDS  = match.group(1)
                tmpDbrLFN = match.group(2)
                # get files in the dataset
                if not tmpDbrDS in dbrDsList:
                    logger.info("Querying files in dataset:%s" % tmpDbrDS)
                    try:
                        tmpList = Client.queryFilesInDataset(tmpDbrDS,False)
                    except:
                        raise ApplicationConfigurationError("ERROR : error while looking up dataset %s. Perhaps this dataset does not exist?"%tmpDbrDS)
                    # append
                    for tmpLFN,tmpVal in tmpList.iteritems():
                        dbrFiles[tmpLFN] = tmpVal
                    dbrDsList.append(tmpDbrDS)
                # check
                if tmpDbrLFN not in dbrFiles:
                    raise ApplicationConfigurationError("ERROR : %s is not in %s"%(tmpDbrLFN,tmpDbrDS))
    return dbrFiles,dbrDsList
Example #2
0
def getDBDatasets(jobO, trf, dbrelease):
    from pandatools import Client

    # get DB datasets
    dbrFiles = {}
    dbrDsList = []
    if trf or dbrelease != '':
        if trf:
            # parse jobO for TRF
            tmpItems = jobO.split()
        else:
            # mimic a trf parameter to reuse following algorithm
            tmpItems = ['%DB=' + dbrelease]
        # look for DBRelease
        for tmpItem in tmpItems:
            match = re.search('%DB=([^:]+):(.+)$', tmpItem)
            if match:
                tmpDbrDS = match.group(1)
                tmpDbrLFN = match.group(2)
                # get files in the dataset
                if not tmpDbrDS in dbrDsList:
                    logger.info("Querying files in dataset:%s" % tmpDbrDS)
                    try:
                        tmpList = Client.queryFilesInDataset(tmpDbrDS, False)
                    except:
                        raise ApplicationConfigurationError(
                            None,
                            "ERROR : error while looking up dataset %s. Perhaps this dataset does not exist?"
                            % tmpDbrDS)
                    # append
                    for tmpLFN, tmpVal in tmpList.iteritems():
                        dbrFiles[tmpLFN] = tmpVal
                    dbrDsList.append(tmpDbrDS)
                # check
                if tmpDbrLFN not in dbrFiles:
                    raise ApplicationConfigurationError(
                        None,
                        "ERROR : %s is not in %s" % (tmpDbrLFN, tmpDbrDS))
    return dbrFiles, dbrDsList
Example #3
0
def printGUIDsWithDatasets(guids):
    """ list datasets by GUIDs
    """
    try:
        from pandatools import Client
    except ImportError:
        if os.environ.has_key('PANDA_SYS'):
            pandapath = os.environ['PANDA_SYS']
        else:
            pandapath = PandaSys
        sys.path.append(pandapath + '/lib/python2.4/site-packages')
        try:
            from pandatools import Client
        except ImportError:
            print "EventLookup failed to import PanDA client, GUID->dataset name resolution disabled"
            return False

    # instantiate curl
    curl = Client._Curl()
    curl.verbose = False
    iLookUp = 0
    guidLfnMap = {}
    checkedDSList = []
    # loop over all GUIDs
    for guid in guids.keys():
        # check existing map to avid redundant lookup
        if guidLfnMap.has_key(guid):
            continue
        iLookUp += 1
        if iLookUp % 20 == 0:
            time.sleep(1)
        # get vuids
        url = Client.baseURLDQ2 + '/ws_content/rpc'
        data = {
            'operation': 'queryDatasetsWithFileByGUID',
            'guid': guid,
            'API': '0_3_0',
            'tuid': Client.MiscUtils.wrappedUuidGen()
        }
        status, out = curl.get(url, data, rucioAccount=True)
        if status != 0 or re.search('Exception', str(out)) != None:
            # failed
            guidLfnMap[guid] = "DQ2 query ERROR: " + str(out)
            continue
        if out == '\x00' or out == ():
            guidLfnMap[
                guid] = "DQ2.queryDatasetsWithFileByGUID() returned no results"
            continue
        tmpVUIDs = list(out)
        # get dataset name
        url = Client.baseURLDQ2 + '/ws_repository/rpc'
        data = {
            'operation': 'queryDatasetByVUIDs',
            'vuids': tmpVUIDs,
            'API': '0_3_0',
            'tuid': Client.MiscUtils.wrappedUuidGen()
        }
        status, out = curl.post(url, data, rucioAccount=True)
        if status != 0 or re.search('Exception', str(out)) != None:
            # failed
            guidLfnMap[guid] = "DQ2 query ERROR: " + str(out)
            continue
        if out == '\x00':
            guidLfnMap[guid] = "DQ2.queryDatasetByVUIDs() returned no results"
            continue
        for tmpDsName in out.keys():
            # ignore junk datasets
            if not (tmpDsName.startswith('panda') or \
                    tmpDsName.startswith('user') or \
                    tmpDsName.startswith('group') or \
                    re.search('_sub\d+$',tmpDsName) != None or \
                    re.search('_dis\d+$',tmpDsName) != None or \
                    re.search('_shadow$',tmpDsName) != None \
                    or tmpDsName in checkedDSList ):
                tmpMap = Client.queryFilesInDataset(tmpDsName)
                for tmpLFN, tmpVal in tmpMap.iteritems():
                    guidLfnMap.setdefault(tmpVal['guid'],
                                          []).append([tmpLFN, tmpDsName])
                checkedDSList.append(tmpDsName)

    for guid in guids.keys():
        print guid, guids[guid], guidLfnMap.setdefault(guid, "")
    return True