def surls2metalink(self, replicas, metalinkFile):
""" Convert list of replicas (of multiple files) to metalink
Input argument, replicas, is dict with guid as key, and a list of surls
Mappings from surl to https turl will come from ddm eventually
to cover surls from remote SEs.
For now just add the mapping for the local SE from copysetup.
"""
site_name = self.site_name
local_se_token = site_name + "_DATADISK"
tolog("local SE token: %s" % (local_se_token))
# self.surl2https_map has key is srm hostname, then tuple of (from,to) regexp replace
dirAcc = getDirectAccessDic(readpar('copysetupin'))
if not dirAcc:
dirAcc = getDirectAccessDic(readpar('copysetup'))
# extract srm host for key
srmhost = None
if dirAcc:
srmhost = self.hostFromSurl(dirAcc['oldPrefix'])
for guid in replicas.keys():
reps = replicas[guid]
tolog("Got replicas=%s for guid=%s" % (str(reps), guid))
try:
token_file = open('token_file', 'r')
except IOError, e:
tolog("!!WARNING!! Failed to open file: %s" % (e))
raise Exception("!!FAILED!!1099!! Cannot open file with token!")
def surls2metalink(self,replicas,metalinkFile):
""" Convert list of replicas (of multiple files) to metalink
Input argument, replicas, is dict with guid as key, and a list of surls
Mappings from surl to https turl will come from ddm eventually
to cover surls from remote SEs.
For now just add the mapping for the local SE from copysetup.
"""
site_name=self.site_name
local_se_token=site_name+"_DATADISK"
tolog("local SE token: %s"%(local_se_token))
# self.surl2https_map has key is srm hostname, then tuple of (from,to) regexp replace
dirAcc = getDirectAccessDic(readpar('copysetupin'))
if not dirAcc:
dirAcc = getDirectAccessDic(readpar('copysetup'))
# extract srm host for key
srmhost=None
if dirAcc:
srmhost = self.hostFromSurl(dirAcc['oldPrefix'])
for guid in replicas.keys():
reps = replicas[guid]
tolog("Got replicas=%s for guid=%s" % (str(reps), guid))
try:
token_file=open('token_file', 'r')
except IOError, e:
tolog ("!!WARNING!! Failed to open file: %s"%(e))
raise Exception("!!FAILED!!1099!! Cannot open file with token!")
def getFileTransferInfo(self, transferType, buildJob):
""" Get all relevant fields related to file transfer """
copysetup = readpar('copysetupin')
# create the direct access dictionary
fileTransferInfo = getDirectAccessDic(copysetup)
# if copysetupin did not contain direct access info, try the copysetup instead
if not fileTransferInfo:
copysetup = readpar('copysetup')
fileTransferInfo = getDirectAccessDic(copysetup)
# should the copytool be used?
useCopyTool = False
useFileStager = False
useDirectAccess = False
oldPrefix = ""
newPrefix = ""
dInfo = None
if fileTransferInfo:
dInfo = True
# no direct access / remote I/O, use standard copytool (copy-to-scratch)
if fileTransferInfo['useCopyTool']:
useCopyTool = True
# do not set the LFC host for file stager
if fileTransferInfo['useFileStager']:
useFileStager = True
if fileTransferInfo['directIn']:
useDirectAccess = True
oldPrefix = fileTransferInfo['oldPrefix']
newPrefix = fileTransferInfo['newPrefix']
# override settings for transferType direct
if transferType == 'direct':
useCopyTool = False
useFileStager = False
useDirectAccess = True
# should pilot create TURL based PFC? (not done here, but setup needs to be aware of it)
# if dInfo and useDirectAccess and oldPrefix == "" and newPrefix == "":
if (transferType == 'direct' or (useFileStager and useDirectAccess)) and (oldPrefix == "" and newPrefix == "") and not buildJob:
# if (transferType == 'direct' or (not useFileStager and useDirectAccess)) and (oldPrefix == "" and newPrefix == ""):
usePFCTurl = True
else:
usePFCTurl = False
# force usePFCTurl for all jobs
if not buildJob and useDirectAccess:
tolog("Forced usePFCTurl (reset old/newPrefix)")
usePFCTurl = True
oldPrefix = ""
newPrefix = ""
if os.environ.get("TestXRootD", 'False') == 'True':
import re
re.sub(r'\/xrootdsetup\.sh', '/xrootdsetup-dev.sh', copysetup)
return dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl
def getFileTransferInfo(self, transferType, buildJob):
""" Get all relevant fields related to file transfer """
copysetup = readpar('copysetupin')
# create the direct access dictionary
fileTransferInfo = getDirectAccessDic(copysetup)
# if copysetupin did not contain direct access info, try the copysetup instead
if not fileTransferInfo:
copysetup = readpar('copysetup')
fileTransferInfo = getDirectAccessDic(copysetup)
# should the copytool be used?
useCopyTool = False
useFileStager = False
useDirectAccess = False
oldPrefix = ""
newPrefix = ""
dInfo = None
if fileTransferInfo:
dInfo = True
# no direct access / remote I/O, use standard copytool (copy-to-scratch)
if fileTransferInfo['useCopyTool']:
useCopyTool = True
# do not set the LFC host for file stager
if fileTransferInfo['useFileStager']:
useFileStager = True
if fileTransferInfo['directIn']:
useDirectAccess = True
oldPrefix = fileTransferInfo['oldPrefix']
newPrefix = fileTransferInfo['newPrefix']
# override settings for transferType direct
if transferType == 'direct':
useCopyTool = False
useFileStager = False
useDirectAccess = True
# should pilot create TURL based PFC? (not done here, but setup needs to be aware of it)
# if dInfo and useDirectAccess and oldPrefix == "" and newPrefix == "":
if (transferType == 'direct' or (useFileStager and useDirectAccess)) and (oldPrefix == "" and newPrefix == "") and not buildJob:
# if (transferType == 'direct' or (not useFileStager and useDirectAccess)) and (oldPrefix == "" and newPrefix == ""):
usePFCTurl = True
else:
usePFCTurl = False
# force usePFCTurl for all jobs
if not buildJob and useDirectAccess:
tolog("Forced usePFCTurl (reset old/newPrefix)")
usePFCTurl = True
oldPrefix = ""
newPrefix = ""
if os.environ.get("TestXRootD", 'False') == 'True':
import re
re.sub(r'\/xrootdsetup\.sh', '/xrootdsetup-dev.sh', copysetup)
return dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl
def getFileTransferInfo(self, transferType, buildJob):
""" Get all relevant fields related to file transfer """
copysetup = readpar('copysetupin')
# create the direct access dictionary
fileTransferInfo = getDirectAccessDic(copysetup)
# if copysetupin did not contain direct access info, try the copysetup instead
if not fileTransferInfo:
copysetup = readpar('copysetup')
fileTransferInfo = getDirectAccessDic(copysetup)
# should the copytool be used?
useCopyTool = False
useFileStager = False
useDirectAccess = False
lfcHost = readpar('lfchost')
oldPrefix = ""
newPrefix = ""
dInfo = None
if fileTransferInfo:
dInfo = True
# no direct access / remote I/O, use standard copytool (copy-to-scratch)
if fileTransferInfo['useCopyTool']:
useCopyTool = True
# do not set the LFC host for file stager
if fileTransferInfo['useFileStager']:
useFileStager = True
if fileTransferInfo['directIn']:
useDirectAccess = True
oldPrefix = fileTransferInfo['oldPrefix']
newPrefix = fileTransferInfo['newPrefix']
# override settings for transferType direct
if transferType == 'direct':
useCopyTool = False
useFileStager = False
useDirectAccess = True
if oldPrefix == "" and newPrefix == "":
lfcHost = ""
# should pilot create TURL based PFC? (not done here, but setup needs to be aware of it)
# if dInfo and useDirectAccess and oldPrefix == "" and newPrefix == "":
if (transferType == 'direct' or (useFileStager and useDirectAccess)) and (oldPrefix == "" and newPrefix == "") and not buildJob:
# if (transferType == 'direct' or (not useFileStager and useDirectAccess)) and (oldPrefix == "" and newPrefix == ""):
usePFCTurl = True
else:
usePFCTurl = False
return dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost
def surls2metalink(self,replicas,metalinkFile):
""" Convert list of replicas (of multiple files) to metalink
Input argument, replicas, is dict with guid as key, and a list of surls
Mappings from surl to https turl will come from ddm eventually
to cover surls from remote SEs.
For now just add the mapping for the local SE from copysetup.
"""
# self.surl2https_map has key is srm hostname, then tuple of (from,to) regexp replace
dirAcc = getDirectAccessDic(readpar('copysetupin'))
if not dirAcc:
dirAcc = getDirectAccessDic(readpar('copysetup'))
# extract srm host for key
if dirAcc:
srmhost = self.hostFromSurl(dirAcc['oldPrefix'])
if srmhost:
self.surl2https_map[srmhost] = (dirAcc['oldPrefix'],dirAcc['newPrefix'])
# Start building metalink
metalink='<?xml version="1.0" encoding="utf-8"?>\n'
metalink+='<metalink version="3.0" generator="Pilot" xmlns="http://www.metalinker.org/">\n'
metalink+='<files>\n'
for guid in replicas.keys():
reps = replicas[guid]
# surl can have __DQ2blah at the end - strip it
name = reps[0].sfn.split('/')[-1]
extindex = name.rfind('__DQ2-')
if extindex > 0: name = name[:extindex]
metalink+='<file name="%s">\n'%name
metalink+='<size>%s</size>'%reps[0].filesize
metalink+='<verification><hash type="adler32">%s</hash></verification>\n'%reps[0].csumvalue
metalink+='<resources>\n'
# if the surl matches a list of https sites, then add a url
for rep in reps:
srmhost = self.hostFromSurl(rep.sfn)
if srmhost in self.surl2https_map.keys():
pair = self.surl2https_map[srmhost]
metalink+='<url type="https" >%s</url>\n'% \
re.sub(pair[0],pair[1],rep.sfn)
else:
tolog("Not found: %s"%rep.sfn)
metalink+='</resources></file>\n'
metalink+='</files></metalink>\n'
print metalink
mlfile = open(metalinkFile,'w')
mlfile.write(metalink)
mlfile.close()
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'local'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-','')
if not path:
tolog('path is empty, using current directory')
path = os.getcwd()
# build setup string
envsetup = self.getEnvsetup(get=True)
# should the root file be copied or read directly by athena?
directIn = False
dInfo = getDirectAccessDic(readpar('copysetupin'))
# if copysetupin did not contain direct access info, try the copysetup instead
if not dInfo:
dInfo = getDirectAccessDic(readpar('copysetup'))
tolog("dInfo: %s" % str(dInfo))
# check if we should use the copytool
if dInfo:
directIn = dInfo['directIn']
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
return 0, self.__pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("not directIn")
# build the get command
_params = ""
if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
csumtype = self.getChecksumType(fchecksum)
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
csumtype = 'md5'
execStr = self.__localget % (envsetup, _params, gpfn, os.path.join(path, lfn))
tolog("Executing command: %s" % (execStr))
report['transferStart'] = time()
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout