コード例 #1
0
def getSEsForSites(ids):
    '''Retrieve the SEs for a site'''

    _refreshToACache()

    #   sites can be a colon seperated list like 'CERN:LYON:ASGC'

    re_srm = re.compile('srm://([^/]+)(/.+)')
    se_dict = {}
    for id in ids:

        sites = _resolveSites(id.upper())
        if not sites:
            logger.error('Site %s not found in TiersOfATLAS', id)
            continue

        for site in sites:
            site_info = ToACache.sites[site]
            if 'srm' not in site_info:
                logger.error('Site %s has no srm info in TiersOfATLAS', site)
                continue
            sitesrm = site_info['srm']
            if sitesrm == '':
                logger.debug('Site %s has no srm info in TiersOfATLAS', site)
                continue
            sitesrm = re.sub('token:*\w*:', '', sitesrm)
            sitesrm = re.sub(':*\d*/srm/managerv2\?SFN=', '', sitesrm)
            match = re_srm.match(sitesrm)
            if not match:
                logger.debug('Cannot extract host from %', sitesrm)
                continue
            se_dict[match.group(1)] = True

    return se_dict.keys()
コード例 #2
0
def getSEsForSites(ids):
    '''Retrieve the SEs for a site'''

    _refreshToACache()

#   sites can be a colon seperated list like 'CERN:LYON:ASGC'

    re_srm = re.compile('srm://([^/]+)(/.+)')
    se_dict = {}
    for id in ids:

        sites = _resolveSites(id.upper())
        if not sites:
            logger.error('Site %s not found in TiersOfATLAS',id)
            continue

        for site in sites:
            site_info = ToACache.sites[site]
            if 'srm' not in site_info: 
                logger.error('Site %s has no srm info in TiersOfATLAS',site)
                continue
            sitesrm = site_info['srm']
            if sitesrm == '': 
                logger.debug('Site %s has no srm info in TiersOfATLAS',site)
                continue
            sitesrm = re.sub('token:*\w*:','', sitesrm)
            sitesrm = re.sub(':*\d*/srm/managerv2\?SFN=','', sitesrm)
            match = re_srm.match(sitesrm)
            if not match:
                logger.debug('Cannot extract host from %',sitesrm)
                continue
            se_dict[match.group(1)] = True

    return se_dict.keys()
コード例 #3
0
def getAllSites(only_lcg=True,
                resolve=False,
                excluded_clouds=[],
                excluded_sites=[],
                blacklist=True):
    '''list all sites defined in TiersOfATLAS'''

    _refreshToACache()
    _refreshCESEInfo()

    sites = []
    if not 'TO' in excluded_clouds:
        sites += getSites('CERN')
    if not 'IT' in excluded_clouds:
        sites += getSites('ITALYSITES')
    if not 'ES' in excluded_clouds:
        sites += getSites('SPAINSITES')
    if not 'FR' in excluded_clouds:
        sites += getSites('FRANCESITES')
    if not 'UK' in excluded_clouds:
        sites += getSites('UKSITES')
    if not 'DE' in excluded_clouds:
        sites += getSites('FZKSITES')
    if not 'NL' in excluded_clouds:
        sites += getSites('NLSITES')
    if not 'TW' in excluded_clouds:
        sites += getSites('TAIWANSITES')
    if not 'CA' in excluded_clouds:
        sites += getSites('CANADASITES')

    if not only_lcg:
        sites += getSites('USASITES')
        sites += getSites('NDGF')

    # exclude sites
    for site in excluded_sites:
        if site in sites:
            sites.remove(site)

    if resolve: sites = _resolveSites(sites)

    # exclude sites - check again after site resolution
    for site in excluded_sites:
        if site in sites:
            sites.remove(site)

    sites.sort()

    if (blacklist):
        for bad_site in CESEInfo['blacklist']:
            try:
                sites.remove(bad_site)
            except ValueError:
                pass

    return sites
コード例 #4
0
def _resolveSites(sites):

    new_sites = []
    for site in sites:
        if site in ToACache.topology:
            new_sites += _resolveSites(ToACache.topology[site])
        else:
            new_sites.append(site)

    return new_sites
コード例 #5
0
def _resolveSites(sites):

    new_sites = []
    for site in sites:
        if site in ToACache.topology:
            new_sites += _resolveSites(ToACache.topology[site])
        else:
            new_sites.append(site)

    return new_sites
コード例 #6
0
def getAllSites(only_lcg=True,resolve=False, excluded_clouds=[], excluded_sites=[], blacklist=True):
    '''list all sites defined in TiersOfATLAS'''

    _refreshToACache()
    _refreshCESEInfo()

    sites = []
    if not 'TO' in excluded_clouds:
        sites += getSites('CERN')
    if not 'IT' in excluded_clouds:
        sites += getSites('ITALYSITES')
    if not 'ES' in excluded_clouds:
        sites += getSites('SPAINSITES')
    if not 'FR' in excluded_clouds:
        sites += getSites('FRANCESITES')
    if not 'UK' in excluded_clouds:
        sites += getSites('UKSITES')
    if not 'DE' in excluded_clouds:
        sites += getSites('FZKSITES')
    if not 'NL' in excluded_clouds:
        sites += getSites('NLSITES')
    if not 'TW' in excluded_clouds:
        sites += getSites('TAIWANSITES')
    if not 'CA' in excluded_clouds:
        sites += getSites('CANADASITES')    

    if not only_lcg:
        sites += getSites('USASITES')
        sites += getSites('NDGF')

    # exclude sites
    for site in excluded_sites:
        if site in sites:
            sites.remove(site)
    
    if resolve: sites = _resolveSites(sites)

    # exclude sites - check again after site resolution
    for site in excluded_sites:
        if site in sites:
            sites.remove(site)

    sites.sort()

    if (blacklist):
        for bad_site in CESEInfo['blacklist']:
            try:
                sites.remove(bad_site)
            except ValueError:
                pass
     
    return sites
コード例 #7
0
def getCEsForSites(ids, excluded_ids=[], CREAM=False, cputime=0):
    '''Retrieve the CEs for a site'''

    _refreshToACache()
    _refreshCESEInfo()

    re_srm = re.compile('srm://([^/]+)(/.+)')
    ce_dict = {}
    for id in ids:

        #       allow the full use of all ToA names as FZKSITES etc.

        sites = _resolveSites([id.upper()])
        if not sites:
            logger.error('Site %s not found in TiersOfATLAS', id)
            continue

        # remove excluded sites
        try:
            excluded_sites = config['ExcludedSites'].split()
        except ConfigError:
            excluded_sites = []

        if excluded_ids:
            excluded_sites = excluded_ids + excluded_sites

        for site in excluded_sites:
            if site in sites:
                logger.warning('Site %s has been excluded.', site)
                sites.remove(site)


#       try to find CEs associated to a site via srm tag and/or associated names tag

        for site in sites:
            site_info = ToACache.sites[site]
            ces = []
            if 'srm' in site_info:
                sitesrm = site_info['srm']
                if sitesrm == '':
                    logger.debug('Site %s has no srm info in TiersOfATLAS',
                                 site)
                    continue
                sitesrm = re.sub('token:*\w*:', '', sitesrm)
                sitesrm = re.sub(':*\d*/srm/managerv2\?SFN=', '', sitesrm)
                match = re_srm.match(sitesrm)
                if not match:
                    logger.debug('Cannot extract host from %s', sitesrm)
                else:
                    try:
                        if CREAM:
                            ces = CESEInfo['se_info'][match.group(
                                1)]['close_creamce']
                            if cputime > 0:
                                for ces_tmp in ces:
                                    cescputime = CESEInfo['creamce_info'][
                                        ces_tmp]['cputime']
                                    if cescputime:
                                        cescputime = int(cescputime)
                                    if cputime < cescputime:
                                        ces = [ces_tmp]
                                        break
                        else:
                            ces = CESEInfo['se_info'][match.group(
                                1)]['close_ce']
                    except KeyError:
                        logger.debug('Did not find CE-SE association for %s',
                                     match.group(1))

            if not ces:
                try:
                    lcg_site = site_info['alternateName'][-1].upper()
                    ces = CESEInfo['lcg_site_info'][lcg_site]
                except Exception:
                    logger.debug(
                        'No CE information on site %s. Maybe it failes the SAM test.',
                        site)

            for ce in ces:
                ce_dict[ce] = True

    return ce_dict.keys()
コード例 #8
0
def getCEsForSites(ids, excluded_ids = [], CREAM = False, cputime=0 ):
    '''Retrieve the CEs for a site'''

    _refreshToACache()
    _refreshCESEInfo()

    re_srm = re.compile('srm://([^/]+)(/.+)')
    ce_dict = {}
    for id in ids:

#       allow the full use of all ToA names as FZKSITES etc.
 
        sites = _resolveSites([id.upper()])
        if not sites:
            logger.error('Site %s not found in TiersOfATLAS',id)
            continue

        # remove excluded sites
        try:
            excluded_sites = config['ExcludedSites'].split()
        except ConfigError:
            excluded_sites = [ ]

        if excluded_ids:
            excluded_sites = excluded_ids + excluded_sites 
        
        for site in excluded_sites:
            if site in sites:
                logger.warning('Site %s has been excluded.',site)
                sites.remove(site)

#       try to find CEs associated to a site via srm tag and/or associated names tag

        for site in sites:
            site_info = ToACache.sites[site] 
            ces = []
            if 'srm' in site_info:
                sitesrm = site_info['srm']
                if sitesrm == '': 
                    logger.debug('Site %s has no srm info in TiersOfATLAS',site)
                    continue
                sitesrm = re.sub('token:*\w*:','', sitesrm)
                sitesrm = re.sub(':*\d*/srm/managerv2\?SFN=','', sitesrm)
                match = re_srm.match(sitesrm)
                if not match:
                    logger.debug('Cannot extract host from %s',sitesrm)
                else:
                    try:
                        if CREAM:
                            ces = CESEInfo['se_info'][match.group(1)]['close_creamce']
                            if cputime > 0:
                                for ces_tmp in ces:
                                    cescputime = CESEInfo['creamce_info'][ces_tmp]['cputime']
                                    if cescputime:
                                        cescputime = int(cescputime)
                                    if cputime < cescputime:
                                        ces = [ ces_tmp ]
                                        break
                        else:
                            ces = CESEInfo['se_info'][match.group(1)]['close_ce']
                    except KeyError:
                        logger.debug('Did not find CE-SE association for %s',match.group(1))

            if not ces:
                try:
                    lcg_site = site_info['alternateName'][-1].upper()
                    ces = CESEInfo['lcg_site_info'][lcg_site]
                except Exception:
                    logger.debug('No CE information on site %s. Maybe it failes the SAM test.',site)

            for ce in ces: 
                ce_dict[ce] = True
                
    return ce_dict.keys()