def get_srm_endpoint(dq2_site_id): ''' Gets the SRM endpoint of a site registered in TiersOfATLAS. @param dq2_site_id is a DQ2 site id @return a dictionary containing the srm endpoint information ''' srm_endpoint_info = {'token':None, 'endpt':None, 'se_host':None, 'se_path':None} re_srm2 = re.compile('^token:(.*):(srm:\/\/.*)\s*$') srm_endpt = TiersOfATLAS.getSiteProperty(dq2_site_id,'srm') if srm_endpt: mat = re_srm2.match(srm_endpt) if mat: # this is a SRMv2 endpoint specification srm_endpoint_info['token'] = mat.group(1) srm_endpoint_info['endpt'] = mat.group(2) endpt_data = urisplit(srm_endpoint_info['endpt']) srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0] srm_endpoint_info['se_path'] = endpt_data[3].replace('SFN=','') else: # this is a SRMv1 endpoint specification srm_endpoint_info['token'] = None srm_endpoint_info['endpt'] = srm_endpt endpt_data = urisplit(srm_endpoint_info['endpt']) srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0] srm_endpoint_info['se_path'] = endpt_data[2] return srm_endpoint_info
def get_srmv2_sites(cloud=None, token=None, debug=False): ''' Gets a list of SRMV2 enabled DDM sites in a given cloud. @param cloud is the ATLAS cloud name @param token restricts the output to only certain srmv2 tokens @param debug indicates if debugging messages are printed @return a list of ATLAS srmv2-enabled site names if token is given, only the site with the specific token type will be selected. ''' srmv2_sites = [] ## a better way of getting all sites within a cloud ## however, it seems there is a bug in DQ2 API so it ## always returns an empty site list. # all_sites = TiersOfATLAS.getSitesInCloud(cloud) ## a bit of hack with non-public DQ2 API interface cache = TiersOfATLAS.ToACache all_sites = [] if not cloud: all_sites = TiersOfATLAS.getAllDestinationSites() else: if cloud == 'T0': return ['CERNPROD'] if cloud not in cache.dbcloud: return [] all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud]) for site in all_sites: srm = TiersOfATLAS.getSiteProperty(site,'srm') # presuming the srm endpoint looks like: # token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ... if srm is not None and srm.find('token') != -1: if token: if srm.split(':')[1] == token: srmv2_sites.append(site) else: srmv2_sites.append(site) return srmv2_sites
def get_srmv2_sites(cloud=None, token=None, debug=False): ''' Gets a list of SRMV2 enabled DDM sites in a given cloud. @param cloud is the ATLAS cloud name @param token restricts the output to only certain srmv2 tokens @param debug indicates if debugging messages are printed @return a list of ATLAS srmv2-enabled site names if token is given, only the site with the specific token type will be selected. ''' srmv2_sites = [] ## a better way of getting all sites within a cloud ## however, it seems there is a bug in DQ2 API so it ## always returns an empty site list. # all_sites = TiersOfATLAS.getSitesInCloud(cloud) ## a bit of hack with non-public DQ2 API interface cache = TiersOfATLAS.ToACache all_sites = [] if not cloud: all_sites = TiersOfATLAS.getAllDestinationSites() else: if cloud == 'T0': return ['CERNPROD'] if cloud not in cache.dbcloud: return [] all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud]) for site in all_sites: srm = TiersOfATLAS.getSiteProperty(site, 'srm') # presuming the srm endpoint looks like: # token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ... if srm is not None and srm.find('token') != -1: if token: if srm.split(':')[1] == token: srmv2_sites.append(site) else: srmv2_sites.append(site) return srmv2_sites
def getSiteProperty(self,seName,attribute): methodName = 'getSiteProperty' try: return self.SC_SUCCEEDED,TiersOfATLAS.getSiteProperty(seName,attribute) except: errtype,errvalue = sys.exc_info()[:2] errCode = self.checkError(errtype) return errCode,'%s : %s %s' % (methodName,errtype.__name__,errvalue)
def get_lfc_host(dq2_site_id): ''' Gets the LFC host of a site registered in TiersOfATLAS. ''' lfc_url = TiersOfATLAS.getLocalCatalog(dq2_site_id) if lfc_url: return lfc_url.split('/')[2][:-1] else: return None
def get_srm_endpoint(dq2_site_id): ''' Gets the SRM endpoint of a site registered in TiersOfATLAS. @param dq2_site_id is a DQ2 site id @return a dictionary containing the srm endpoint information ''' srm_endpoint_info = { 'token': None, 'endpt': None, 'se_host': None, 'se_path': None } re_srm2 = re.compile('^token:(.*):(srm:\/\/.*)\s*$') srm_endpt = TiersOfATLAS.getSiteProperty(dq2_site_id, 'srm') if srm_endpt: mat = re_srm2.match(srm_endpt) if mat: # this is a SRMv2 endpoint specification srm_endpoint_info['token'] = mat.group(1) srm_endpoint_info['endpt'] = mat.group(2) endpt_data = urisplit(srm_endpoint_info['endpt']) srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0] srm_endpoint_info['se_path'] = endpt_data[3].replace('SFN=', '') else: # this is a SRMv1 endpoint specification srm_endpoint_info['token'] = None srm_endpoint_info['endpt'] = srm_endpt endpt_data = urisplit(srm_endpoint_info['endpt']) srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0] srm_endpoint_info['se_path'] = endpt_data[2] return srm_endpoint_info
locnum = 0 for i in locrange: locnum += len(locations[i]) if locnum == 0: print "ERROR no location" sys.exit(0) lfn_guid = {} for guid, info in contents.iteritems(): lfn_guid[info["lfn"]] = guid allLFCs = [] for i in locrange: for location in locations[i]: l = TiersOfATLAS.getLocalCatalog(location) if l and l not in allLFCs and l.startswith("lfc") and l not in removefromlfclist: allLFCs.append(l) status, guidReplicas, guidSizes, guidmd5sum = getinputreplicas(lfn_guid, allLFCs) print guidReplicas locations_srm = {} for i in locrange: for location in locations[i]: try: if "srm" in TiersOfATLAS.ToACache.sites[location]: tempsrm = TiersOfATLAS.ToACache.sites[location]["srm"] tempsrm = re.sub("token:*\w*:", "", tempsrm) tempsrm = re.sub(":*\d*/srm/managerv2\?SFN=", "", tempsrm) print tempsrm
def stripSite(site): dq2alternatename = TiersOfATLAS.getSiteProperty(site,'alternateName') if not dq2alternatename: return site else: return dq2alternatename[0]
def main(): usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option("-q", action="store_true", default=False, help="quiet mode", dest="quiet") parser.add_option("-d", action="store_true", default=False, help="debug mode", dest="debug") (options, args) = parser.parse_args() if len(args) != 0: parser.error("incorrect number of arguments") return 1 loglevel = 'INFO' if options.quiet: loglevel = 'WARNING' if options.debug: loglevel = 'DEBUG' logger = logging.getLogger() logger.setLevel(logging._levelNames[loglevel]) fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s' formatter = logging.Formatter(fmt, '%d %b %T') handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.handlers = [] logger.addHandler(handler) proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out' labels = TiersOfATLAS.getSites('UKSITES') #labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK'] #labels = ['UKI-SCOTGRID-GLASGOW_MCDISK'] #labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK'] #labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK'] #labels = [ # 'UKI-LT2-IC-HEP_PRODDISK', # 'UKI-LT2-IC-HEP_MCDISK', # 'UKI-LT2-IC-HEP_DATADISK', # 'UKI-LT2-IC-HEP_HOTDISK', # ] #labels = ['UKI-SOUTHGRID-SUSX_PRODDISK'] try: labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK') except: pass try: labels.remove('RAL-LCG2_PPSDATADISK') except: pass for label in labels: # if 'SUSX' not in label: continue srm = TiersOfATLAS.getSiteProperty(label, 'srm') domain = TiersOfATLAS.getSiteProperty(label, 'domain') srmmatch = _SRMMATCH.match(srm) srmtok = srmmatch.group(1) srmpath = srmmatch.group(2) srmhost = srmmatch.group(3) cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % ( proxy, srmpath, srmtok) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: tokmatch = _TOKENMATCH.match(output) if tokmatch: tokenid = tokmatch.group(1) msg = "Found token ID %s: %s" % (srmtok, tokenid) logging.debug(msg) else: msg = "Token ID not found for %s at %s" % (srmtok, label) logging.warn(msg) continue else: msg = "Cannot get token ID using cmd %s" % cmd logging.warn(msg) continue # replace this with lcg-stmd cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % ( proxy, tokenid, srmpath) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: spacematch = _SPACEMATCH.match(output) if spacematch: totsize = spacematch.group(1) freesize = spacematch.group(3) else: msg = "Cannot parse output: %s" % cmd logging.warn(msg) continue else: msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid logging.warn(msg) continue msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize) logging.info(msg) vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) # _curl.setopt(pycurl.VERBOSE, True) try: _curl.perform() _buffer.seek(0) if _curl.getinfo(pycurl.HTTP_CODE) != 200: msg = "failed: %s%s" % (_UPDATEURL, fields) logging.debug(msg) # read from start of buffer _buffer.seek(0) logging.debug(_buffer.read()) for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise # hack for RAL FARM area label = 'RAL-LCG2_RALFARM' totsize = '102000000000000.0' freesize = '0.0' vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) try: _curl.perform() _buffer.seek(0) code = _curl.getinfo(pycurl.HTTP_CODE) if code == 500: for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise
#!/usr/bin/env python import os from dq2.clientapi.DQ2 import DQ2 from dq2.info import TiersOfATLAS from dq2.info.TiersOfATLAS import _refreshToACache, ToACache _refreshToACache() dq2localid = os.environ['DQ2_LOCAL_SITE_ID'] dq2localid_alternatename = TiersOfATLAS.getSiteProperty(dq2localid,'alternateName') if not dq2localid_alternatename: try: dq2localid_alternatename = dq2localid.split("_")[0] except: dq2localid_alternatename = dq2localid db_dataset = os.environ['ATLAS_DBRELEASE'] dq2=DQ2() db_locations = dq2.listDatasetReplicas(db_dataset).values()[0][1] print "db_locations = ", db_locations db_site = dq2localid for sitename in TiersOfATLAS.getAllSources(): if TiersOfATLAS.getSiteProperty(sitename,'alternateName')==dq2localid_alternatename and sitename in db_locations: db_site = sitename outFile = open('db_dq2localid.txt','w') outFile.write('%s\n' % db_site) outFile.close()
import commands import sys from dq2.info import TiersOfATLAS outputFile = open('infoSites.txt','a') site=sys.argv[1] if site=="SWT2_CPB_DATADISK": outputFile.write(site) outputFile.write('\n') outputFile.write('unknown\n') outputFile.write('unknown\n') exit() srm = TiersOfATLAS.getSiteProperty(site, 'srm') endpoint = "/".join(":".join(srm.split(':')[2:5]).split('/')[:3]) s, o = commands.getstatusoutput('srmping -debug=True -2 -retry_num=1 %s' % endpoint) outputFile.write(site) outputFile.write('\n') foundType=0 foundversion=0 if s == 0: for line in o.split('\n'): if line.find('backend_type') > -1: outputFile.write(line.replace('backend_type:','',1)) outputFile.write('\n') foundType=1 if line.find('backend_version') > -1: outputFile.write(line.strip('backend_version:')) outputFile.write('\n') foundversion=1
def stripSite(site): dq2alternatename = TiersOfATLAS.getSiteProperty(site, 'alternateName') if not dq2alternatename: return site else: return dq2alternatename[0]
def resolve_dq2_local_site_id(ds_locations, site_domain, se_hostname, force_siteid_domain={}, force_siteid_se={}): ''' resolves the DQ2_LOCAL_SITE_ID based on 3 factors: - the dataset location list - the site domain - the default SE host - the dictionaries containing the local_site_id enforcement The "force_siteid_domain" gives a list in which the matched site_domain (regx match) will be limited to the given list of DQ2 site ids. The "force_siteid_se" gives a list in which the matched se_hostname (regx match) will be limited to the given list of DQ2 site ids. it returns a final DQ2 site id which is local and containing the dataset replica or an empty string if there is no matched DQ2 site id. ''' dq2_local_site_id = '' toa_check = True # get dq2 site ids matching the site_domain and se_hostname dq2_local_ids = [] # checking if the given site_domain in the enforcement dictionary if force_siteid_domain: for mydomain, mysites in force_siteid_domain.items(): re_domain = re.compile(mydomain) if re_domain.match(site_domain): dq2_local_ids = mysites toa_check = False break # checking if the given se_hostname in the enforcement dictionary if force_siteid_se: for myse, mysites in force_siteid_se.items(): re_se = re.compile(myse) if re_se.match(se_hostname): dq2_local_ids = mysites toa_check = False break # if enforcement is never applied, do the detection for dq2_local_ids via ToA if toa_check: alternateNameDict = {} for sitename in TiersOfATLAS.getAllSources(): # compose the altname:siteid dictionary altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName') if altnames: for altname in altnames: if altname not in alternateNameDict: alternateNameDict[altname] = [] alternateNameDict[altname].append(sitename) # First search for srm dq2srm = TiersOfATLAS.getSiteProperty(sitename,'srm') if dq2srm and dq2srm.find(se_hostname)>=0: dq2_local_ids.append(sitename) # Second search for domainname dq2domain = TiersOfATLAS.getSiteProperty(sitename,'domain') if dq2domain and dq2domain.find(site_domain)>=0: dq2_local_ids.append(sitename) # Thirdly search for sites with the same alternateName more_dq2_local_ids = [] for sitename in dq2_local_ids: altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName') if altnames: for altname in altnames: if altname in alternateNameDict: more_dq2_local_ids += alternateNameDict[altname] dq2_local_ids += more_dq2_local_ids # resolving the best location according to the dataset locations # - get the common part between dq2_local_ids and ds_locations # - pick the first one in the common part candidates = list(set(dq2_local_ids) & set(ds_locations)) print >> sys.stdout, str(candidates) if candidates: ## pick up the site whose corresponding se is matching the se_hostname for c in candidates: srm_info = get_srm_endpoint(c) if srm_info['se_host'] == se_hostname: dq2_local_site_id = c break ## otherwise, take the first candidate in the list if not dq2_local_site_id: dq2_local_site_id = candidates[0] return dq2_local_site_id
def resolve_dq2_local_site_id(ds_locations, site_domain, se_hostname, force_siteid_domain={}, force_siteid_se={}): ''' resolves the DQ2_LOCAL_SITE_ID based on 3 factors: - the dataset location list - the site domain - the default SE host - the dictionaries containing the local_site_id enforcement The "force_siteid_domain" gives a list in which the matched site_domain (regx match) will be limited to the given list of DQ2 site ids. The "force_siteid_se" gives a list in which the matched se_hostname (regx match) will be limited to the given list of DQ2 site ids. it returns a final DQ2 site id which is local and containing the dataset replica or an empty string if there is no matched DQ2 site id. ''' dq2_local_site_id = '' toa_check = True # get dq2 site ids matching the site_domain and se_hostname dq2_local_ids = [] # checking if the given site_domain in the enforcement dictionary if force_siteid_domain: for mydomain, mysites in force_siteid_domain.items(): re_domain = re.compile(mydomain) if re_domain.match(site_domain): dq2_local_ids = mysites toa_check = False break # checking if the given se_hostname in the enforcement dictionary if force_siteid_se: for myse, mysites in force_siteid_se.items(): re_se = re.compile(myse) if re_se.match(se_hostname): dq2_local_ids = mysites toa_check = False break # if enforcement is never applied, do the detection for dq2_local_ids via ToA if toa_check: alternateNameDict = {} for sitename in TiersOfATLAS.getAllSources(): # compose the altname:siteid dictionary altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName') if altnames: for altname in altnames: if altname not in alternateNameDict: alternateNameDict[altname] = [] alternateNameDict[altname].append(sitename) # First search for srm dq2srm = TiersOfATLAS.getSiteProperty(sitename, 'srm') if dq2srm and dq2srm.find(se_hostname) >= 0: dq2_local_ids.append(sitename) # Second search for domainname dq2domain = TiersOfATLAS.getSiteProperty(sitename, 'domain') if dq2domain and dq2domain.find(site_domain) >= 0: dq2_local_ids.append(sitename) # Thirdly search for sites with the same alternateName more_dq2_local_ids = [] for sitename in dq2_local_ids: altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName') if altnames: for altname in altnames: if altname in alternateNameDict: more_dq2_local_ids += alternateNameDict[altname] dq2_local_ids += more_dq2_local_ids # resolving the best location according to the dataset locations # - get the common part between dq2_local_ids and ds_locations # - pick the first one in the common part candidates = list(set(dq2_local_ids) & set(ds_locations)) print >> sys.stdout, str(candidates) if candidates: ## pick up the site whose corresponding se is matching the se_hostname for c in candidates: srm_info = get_srm_endpoint(c) if srm_info['se_host'] == se_hostname: dq2_local_site_id = c break ## otherwise, take the first candidate in the list if not dq2_local_site_id: dq2_local_site_id = candidates[0] return dq2_local_site_id
if job.computingSite in ['', None, 'NULL']: print 'computingSite is not yet defined' sys.exit(0) siteSpec = siteMapper.getSite(job.computingSite) for file in job.Files: if file.type in ['output', 'log']: file.GUID = commands.getoutput('uuidgen') if job.computingSite == file.destinationSE and \ siteSpec.setokens_output.has_key(file.destinationDBlockToken): tmpSrcDDM = siteSpec.setokens_output[file.destinationDBlockToken] else: tmpSrcDDM = siteMapper.getSite(job.computingSite).ddm_output srm = TiersOfATLAS.getSiteProperty(tmpSrcDDM, 'srm') srm = re.sub('^token:[^:]+:', '', srm) xml += """ <File ID="%s"> <logical> <lfn name="%s"/> </logical> <metadata att_name="surl" att_value="%s/user.elmsheus/user.elmsheus.hc10006029.ANALY_LONG_BNL_ATLAS.1312433204.e0b.8181.ANALY_LONG_BNL_ATLAS/%s"/> <metadata att_name="fsize" att_value="127340"/> <metadata att_name="md5sum" att_value="03cea4013bdb9f2e44050449b6ebf079"/> </File>""" % (file.GUID, file.lfn, srm, file.lfn) xml += """ </POOLFILECATALOG> """
def main(): usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option("-q", action="store_true", default=False, help="quiet mode", dest="quiet") parser.add_option("-d", action="store_true", default=False, help="debug mode", dest="debug") (options, args) = parser.parse_args() if len(args) != 0: parser.error("incorrect number of arguments") return 1 loglevel = 'INFO' if options.quiet: loglevel = 'WARNING' if options.debug: loglevel = 'DEBUG' logger = logging.getLogger() logger.setLevel(logging._levelNames[loglevel]) fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s' formatter = logging.Formatter(fmt, '%d %b %T') handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.handlers = [] logger.addHandler(handler) proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out' labels = TiersOfATLAS.getSites('UKSITES') #labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK'] #labels = ['UKI-SCOTGRID-GLASGOW_MCDISK'] #labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK'] #labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK'] #labels = [ # 'UKI-LT2-IC-HEP_PRODDISK', # 'UKI-LT2-IC-HEP_MCDISK', # 'UKI-LT2-IC-HEP_DATADISK', # 'UKI-LT2-IC-HEP_HOTDISK', # ] #labels = ['UKI-SOUTHGRID-SUSX_PRODDISK'] try: labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK') except: pass try: labels.remove('RAL-LCG2_PPSDATADISK') except: pass for label in labels: # if 'SUSX' not in label: continue srm = TiersOfATLAS.getSiteProperty(label, 'srm') domain = TiersOfATLAS.getSiteProperty(label, 'domain') srmmatch = _SRMMATCH.match(srm) srmtok = srmmatch.group(1) srmpath = srmmatch.group(2) srmhost = srmmatch.group(3) cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % (proxy, srmpath, srmtok) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: tokmatch = _TOKENMATCH.match(output) if tokmatch: tokenid = tokmatch.group(1) msg = "Found token ID %s: %s" % (srmtok, tokenid) logging.debug(msg) else: msg = "Token ID not found for %s at %s" % (srmtok, label) logging.warn(msg) continue else: msg = "Cannot get token ID using cmd %s" % cmd logging.warn(msg) continue # replace this with lcg-stmd cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % (proxy, tokenid, srmpath) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: spacematch = _SPACEMATCH.match(output) if spacematch: totsize = spacematch.group(1) freesize = spacematch.group(3) else: msg = "Cannot parse output: %s" % cmd logging.warn(msg) continue else: msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid logging.warn(msg) continue msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize) logging.info(msg) vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) # _curl.setopt(pycurl.VERBOSE, True) try: _curl.perform() _buffer.seek(0) if _curl.getinfo(pycurl.HTTP_CODE) != 200: msg = "failed: %s%s" % (_UPDATEURL, fields) logging.debug(msg) # read from start of buffer _buffer.seek(0) logging.debug(_buffer.read()) for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise # hack for RAL FARM area label= 'RAL-LCG2_RALFARM' totsize = '102000000000000.0' freesize = '0.0' vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) try: _curl.perform() _buffer.seek(0) code = _curl.getinfo(pycurl.HTTP_CODE) if code == 500: for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise
def getAvailableFiles(self,datasetSpec,siteEndPointMap,siteMapper,ngGroup=[],checkLFC=False): # make logger methodName = 'getAvailableFiles' methodName += ' <datasetID={0}>'.format(datasetSpec.datasetID) tmpLog = MsgWrapper(logger,methodName) tmpLog.info('start datasetName={0}'.format(datasetSpec.datasetName)) try: # list of NG endpoints ngEndPoints = [] if 1 in ngGroup: ngEndPoints += ['_SCRATCHDISK$','_LOCALGROUPDISK$','_LOCALGROUPTAPE$','_USERDISK$', '_DAQ$','_TMPDISK$','_TZERO$','_GRIDFTP$','MOCKTEST$'] if 2 in ngGroup: ngEndPoints += ['_LOCALGROUPTAPE$', '_DAQ$','_TMPDISK$','_TZERO$','_GRIDFTP$','MOCKTEST$'] # get all associated endpoints siteAllEndPointsMap = {} for siteName,endPointPattList in siteEndPointMap.iteritems(): # get all endpoints matching with patterns allEndPointList = [] for endPointPatt in endPointPattList: if '*' in endPointPatt: # wildcard endPointPatt = endPointPatt.replace('*','.*') for endPointToA in TiersOfATLAS.getAllDestinationSites(): if re.search('^'+endPointPatt+'$',endPointToA) != None: if not endPointToA in allEndPointList: allEndPointList.append(endPointToA) else: # normal endpoint if endPointPatt in TiersOfATLAS.getAllDestinationSites() and \ not endPointPatt in allEndPointList: allEndPointList.append(endPointPatt) # get associated endpoints siteAllEndPointsMap[siteName] = [] for endPoint in allEndPointList: # append if not self.checkNGEndPoint(endPoint,ngEndPoints) and \ not endPoint in siteAllEndPointsMap[siteName]: siteAllEndPointsMap[siteName].append(endPoint) else: # already checked continue # get alternate name altName = TiersOfATLAS.getSiteProperty(endPoint,'alternateName') if altName != None and altName != ['']: for assEndPoint in TiersOfATLAS.resolveGOC({altName[0]:None})[altName[0]]: if not assEndPoint in siteAllEndPointsMap[siteName] and \ not self.checkNGEndPoint(assEndPoint,ngEndPoints): siteAllEndPointsMap[siteName].append(assEndPoint) # get replica map tmpStat,tmpOut = self.listDatasetReplicas(datasetSpec.datasetName) if tmpStat != self.SC_SUCCEEDED: tmpLog.error('faild to get dataset replicas with {0}'.format(tmpOut)) raise tmpStat,tmpOut datasetReplicaMap = tmpOut # collect SE, LFC hosts, storage path, storage type lfcSeMap = {} storagePathMap = {} completeReplicaMap = {} siteHasCompleteReplica = False for siteName,allEndPointList in siteAllEndPointsMap.iteritems(): tmpLfcSeMap = {} tmpStoragePathMap = {} tmpSiteSpec = siteMapper.getSite(siteName) for tmpEndPoint in allEndPointList: # storage type if TiersOfATLAS.isTapeSite(tmpEndPoint): storageType = 'localtape' else: storageType = 'localdisk' # no scan when site has complete replicas if datasetReplicaMap.has_key(tmpEndPoint) and datasetReplicaMap[tmpEndPoint][-1]['found'] != None \ and datasetReplicaMap[tmpEndPoint][-1]['total'] == datasetReplicaMap[tmpEndPoint][-1]['found']: completeReplicaMap[tmpEndPoint] = storageType siteHasCompleteReplica = True # no LFC scan for many-time datasets if datasetSpec.isManyTime(): continue # get LFC lfc = TiersOfATLAS.getLocalCatalog(tmpEndPoint) # add map if not tmpLfcSeMap.has_key(lfc): tmpLfcSeMap[lfc] = [] # get SE seStr = TiersOfATLAS.getSiteProperty(tmpEndPoint, 'srm') tmpMatch = re.search('://([^:/]+):*\d*/',seStr) if tmpMatch != None: se = tmpMatch.group(1) if not se in tmpLfcSeMap[lfc]: tmpLfcSeMap[lfc].append(se) else: tmpLog.error('faild to extract SE from %s for %s:%s' % \ (seStr,siteName,tmpEndPoint)) # get SE + path seStr = TiersOfATLAS.getSiteProperty(tmpEndPoint, 'srm') tmpMatch = re.search('(srm://.+)$',seStr) if tmpMatch == None: tmpLog.error('faild to extract SE+PATH from %s for %s:%s' % \ (seStr,siteName,tmpEndPoint)) continue # add full path to storage map tmpSePath = tmpMatch.group(1) tmpStoragePathMap[tmpSePath] = {'siteName':siteName,'storageType':storageType} # add compact path tmpSePath = re.sub('(:\d+)*/srm/[^\?]+\?SFN=','',tmpSePath) tmpStoragePathMap[tmpSePath] = {'siteName':siteName,'storageType':storageType} # add to map to trigger LFC scan if complete replica is missing at the site if DataServiceUtils.isCachedFile(datasetSpec.datasetName,tmpSiteSpec): pass elif not siteHasCompleteReplica or checkLFC: for tmpKey,tmpVal in tmpLfcSeMap.iteritems(): if not lfcSeMap.has_key(tmpKey): lfcSeMap[tmpKey] = [] lfcSeMap[tmpKey] += tmpVal for tmpKey,tmpVal in tmpStoragePathMap.iteritems(): storagePathMap[tmpKey] = tmpVal # collect GUIDs and LFNs fileMap = {} lfnMap = {} lfnFileSepcMap = {} scopeMap = {} for tmpFile in datasetSpec.Files: fileMap[tmpFile.GUID] = tmpFile.lfn lfnMap[tmpFile.lfn] = tmpFile lfnFileSepcMap[tmpFile.lfn] = tmpFile scopeMap[tmpFile.lfn] = tmpFile.scope # get SURLs surlMap = {} for lfcHost,seList in lfcSeMap.iteritems(): tmpLog.debug('lookup in LFC:{0} for {1}'.format(lfcHost,str(seList))) tmpStat,tmpRetMap = self.getSURLsFromLFC(fileMap,lfcHost,seList,scopes=scopeMap) tmpLog.debug(str(tmpStat)) if tmpStat != self.SC_SUCCEEDED: raise RuntimeError,tmpRetMap for lfn,surls in tmpRetMap.iteritems(): if not surlMap.has_key(lfn): surlMap[lfn] = surls else: surlMap[lfn] += surls # make return returnMap = {} for siteName,allEndPointList in siteAllEndPointsMap.iteritems(): # set default return values if not returnMap.has_key(siteName): returnMap[siteName] = {'localdisk':[],'localtape':[],'cache':[],'remote':[]} # loop over all files tmpSiteSpec = siteMapper.getSite(siteName) # check if the file is cached if DataServiceUtils.isCachedFile(datasetSpec.datasetName,tmpSiteSpec): for tmpFileSpec in datasetSpec.Files: # add to cached file list returnMap[siteName]['cache'].append(tmpFileSpec) # complete replicas if not checkLFC: for tmpEndPoint in allEndPointList: if completeReplicaMap.has_key(tmpEndPoint): storageType = completeReplicaMap[tmpEndPoint] returnMap[siteName][storageType] += datasetSpec.Files # loop over all available LFNs avaLFNs = surlMap.keys() avaLFNs.sort() for tmpLFN in avaLFNs: tmpFileSpec = lfnFileSepcMap[tmpLFN] # loop over all SURLs for tmpSURL in surlMap[tmpLFN]: for tmpSePath in storagePathMap.keys(): # check SURL if tmpSURL.startswith(tmpSePath): # add siteName = storagePathMap[tmpSePath]['siteName'] storageType = storagePathMap[tmpSePath]['storageType'] if not tmpFileSpec in returnMap[siteName][storageType]: returnMap[siteName][storageType].append(tmpFileSpec) break # dump dumpStr = '' for siteName,storageTypeFile in returnMap.iteritems(): dumpStr += '{0}:('.format(siteName) for storageType,fileList in storageTypeFile.iteritems(): dumpStr += '{0}:{1},'.format(storageType,len(fileList)) dumpStr = dumpStr[:-1] dumpStr += ') ' dumpStr= dumpStr[:-1] tmpLog.debug(dumpStr) # return tmpLog.info('done') return self.SC_SUCCEEDED,returnMap except: errtype,errvalue = sys.exc_info()[:2] errMsg = 'failed with {0} {1}'.format(errtype.__name__,errvalue) tmpLog.error(errMsg) return self.SC_FAILED,'{0}.{1} {2}'.format(self.__class__.__name__,methodName,errMsg)
locnum = 0 for i in locrange: locnum += len(locations[i]) if locnum == 0: print 'ERROR no location' sys.exit(0) lfn_guid = {} for guid, info in contents.iteritems(): lfn_guid[info['lfn']] = guid allLFCs = [] for i in locrange: for location in locations[i]: l = TiersOfATLAS.getLocalCatalog(location) if l and l not in allLFCs and l.startswith( 'lfc') and l not in removefromlfclist: allLFCs.append(l) status, guidReplicas, guidSizes, guidmd5sum = getinputreplicas( lfn_guid, allLFCs) print guidReplicas locations_srm = {} for i in locrange: for location in locations[i]: try: if 'srm' in TiersOfATLAS.ToACache.sites[location]: tempsrm = TiersOfATLAS.ToACache.sites[location]['srm'] tempsrm = re.sub('token:*\w*:', '', tempsrm)
def findLostFiles(self,datasetName,fileMap): methodName = 'findLostFiles' methodName += ' <datasetName={0}>'.format(datasetName) tmpLog = MsgWrapper(logger,methodName) tmpLog.info('start') try: # get replicas tmpStat,tmpOut = self.listDatasetReplicas(datasetName) if tmpStat != self.SC_SUCCEEDED: tmpLog.error('faild to get dataset replicas with {0}'.format(tmpOut)) raise tmpStat,tmpOut # check if complete replica is available hasCompReplica = False datasetReplicaMap = tmpOut for tmpEndPoint in datasetReplicaMap.keys(): if datasetReplicaMap[tmpEndPoint][-1]['found'] != None and \ datasetReplicaMap[tmpEndPoint][-1]['total'] == datasetReplicaMap[tmpEndPoint][-1]['found']: hasCompReplica = True break # no lost files if hasCompReplica: tmpLog.info('done with no lost files') self.SC_SUCCEEDED,{} # get LFNs and scopes lfnMap = {} scopeMap = {} for tmpGUID in fileMap.keys(): tmpLFN = fileMap[tmpGUID]['lfn'] lfnMap[tmpGUID] = tmpLFN scopeMap[tmpLFN] = fileMap[tmpGUID]['scope'] # get LFC and SE lfcSeMap = {} for tmpEndPoint in datasetReplicaMap.keys(): # get LFC lfc = TiersOfATLAS.getLocalCatalog(tmpEndPoint) # add map if not lfcSeMap.has_key(lfc): lfcSeMap[lfc] = [] # get SE seStr = TiersOfATLAS.getSiteProperty(tmpEndPoint, 'srm') tmpMatch = re.search('://([^:/]+):*\d*/',seStr) if tmpMatch != None: se = tmpMatch.group(1) if not se in lfcSeMap[lfc]: lfcSeMap[lfc].append(se) # get SURLs for lfcHost,seList in lfcSeMap.iteritems(): tmpStat,tmpRetMap = self.getSURLsFromLFC(lfnMap,lfcHost,seList,scopes=scopeMap) if tmpStat != self.SC_SUCCEEDED: tmpLog.error('faild to get SURLs with {0}'.format(tmpRetMap)) raise tmpStat,tmpRetMap # look for missing files newLfnMap = {} for tmpGUID,tmpLFN in lfnMap.iteritems(): if not tmpLFN in tmpRetMap: newLfnMap[tmpGUID] = tmpLFN lfnMap = newLfnMap tmpLog.info('done with lost '+','.join(str(tmpLFN) for tmpLFN in lfnMap.values())) return self.SC_SUCCEEDED,lfnMap except: errtype,errvalue = sys.exc_info()[:2] errCode = self.checkError(errtype) errMsg = '{0} {1}'.format(errtype.__name__,errvalue) tmpLog.error(errMsg) return errCode,'{0} : {1}'.format(methodName,errMsg)
if job.computingSite in ['',None,'NULL']: print 'computingSite is not yet defined' sys.exit(0) siteSpec = siteMapper.getSite(job.computingSite) for file in job.Files: if file.type in ['output','log']: file.GUID = commands.getoutput('uuidgen') if job.computingSite == file.destinationSE and \ siteSpec.setokens.has_key(file.destinationDBlockToken): tmpSrcDDM = siteSpec.setokens[file.destinationDBlockToken] else: tmpSrcDDM = siteMapper.getSite(job.computingSite).ddm srm = TiersOfATLAS.getSiteProperty(tmpSrcDDM,'srm') srm = re.sub('^token:[^:]+:','',srm) xml += """ <File ID="%s"> <logical> <lfn name="%s"/> </logical> <metadata att_name="surl" att_value="%s/user.elmsheus/user.elmsheus.hc10006029.ANALY_LONG_BNL_ATLAS.1312433204.e0b.8181.ANALY_LONG_BNL_ATLAS/%s"/> <metadata att_name="fsize" att_value="127340"/> <metadata att_name="md5sum" att_value="03cea4013bdb9f2e44050449b6ebf079"/> </File>""" % (file.GUID,file.lfn,srm,file.lfn) xml += """ </POOLFILECATALOG> """
if (debug): sys.stderr.write("Using DQ2 clients\n") if (location): fe=getConfig(conffile,location,'FRONTIER_SERVER') if (not fe): frc,fe=getFrontierEnvDQ2(location) if (not fe): frc,fe=getFrontierEnvDQ2(location.upper()) if (frc == 0 and fe): print fe rc += frc else: for gn in fsquidmap.keys(): frc,fe=getFrontierEnvDQ2(gn) print gn,fe rc += frc elif (mode == 'listproto'): if (ToA): if (location): seinfo = ToA.getSiteProperty(location.upper(), 'seinfo') elif (AGIS): seinfo = a.get_seinfo_entry()[location.upper()] else: sys.stderr.write("No AGIS or DQ2 clients available\n") if (seinfo and seinfo.has_key('protocols')): for p in seinfo['protocols']: print p[0] elif (mode == 'regexp'): if (ToA): if (location): seinfo = ToA.getSiteProperty(location.upper(), 'seinfo') elif (AGIS): seinfo = a.get_seinfo_entry()[location.upper()] else: sys.stderr.write("No AGIS or DQ2 clients available\n")