def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5*3600)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'dCache', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time()
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if not self.isFileStaged(_setup_str, loc_pfn):
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (loc_pfn)
self.prepareReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", ftype="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
telapsed = 0
try:
# t0 = time()
# s, cout = commands.getstatusoutput(_cmd_str)
# cerr = ""
# telapsed = time() - t0
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
print "DEBUG: s = ", s, type(s)
print "DEBUG: telapsed = ", telapsed, type(telapsed)
print "DEBUG: cout = ", cout, type(cout)
print "DEBUG: cerr = ", cerr, type(cerr)
cout = self.filter_text(cout)
cerr = self.filter_text(cerr)
if not self.is_number(s):
s = 1
if not self.is_number(telapsed):
telapsed = 0
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % (e))
s = 1
o = self.filter_text(e)
telapsed = timeout
# write traceback info to stderr
import traceback
exc, msg, tb = sys.exc_info()
traceback.print_tb(tb)
_pilotErrorDiag = "Unexpected exception: %s" % (get_exc_plus())
tolog("!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag))
print "!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag)
def put_data(self, pfn, ddm_storage, fsize=0, fchecksum=0, dsname="", extradirs="", **pdict):
""" Copy all output file to the local SE """
error = PilotErrors()
pilotErrorDiag = ""
tolog("put_data() got ddm_storage=%s" % (ddm_storage))
# Get input parameters from pdict
lfn = pdict.get("lfn", "")
guid = pdict.get("guid", "")
analJob = pdict.get("analJob", False)
experiment = pdict.get("experiment", "")
# get the Rucio tracing report
report = self.getStubTracingReport(pdict["report"], "castor", lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup()
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport("RFCP_FAIL", report)
return self.put_data_retfail(ec, pilotErrorDiag)
# get the experiment object
thisExperiment = getExperiment(experiment)
# do we have a valid proxy?
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup, limit=2)
if s != 0:
self.prepareReport("PROXYFAIL", report)
return self.put_data_retfail(s, pilotErrorDiag)
filename = pfn.split("/")[-1]
# the current file
report["filename"] = lfn
# guid
report["guid"] = guid.replace("-", "")
# Destination is the top level Castor store area. Append a subdirectory which is first two fields of dsname, or 'other'
destination = ""
if not analJob:
# seprodpath can have a complex structure in case of space tokens
# although currently not supported in this site mover, prepare the code anyway
# (use the first list item only)
destination = self.getDirList(readpar("seprodpath"))[0]
if destination == "":
tolog("!!WARNING!!2999!! seprodpath not defined, using sepath")
destination = readpar("sepath")
tolog("Going to store production job output")
else:
destination = readpar("sepath")
tolog("Going to store analysis job output")
if destination == "":
pilotErrorDiag = "put_data destination path in SE not defined"
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport("DEST_PATH_UNDEF", report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
else:
tolog("destination: %s" % (destination))
if dsname == "":
pilotErrorDiag = "Dataset name not specified to put_data"
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport("NO_DSN", report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
# else:
# dsname = self.remove_sub(dsname)
# tolog("dsname: %s" % (dsname))
lfcpath, pilotErrorDiag = self.getLFCPath(analJob)
if lfcpath == "":
self.prepareReport("LFC_PATH_FAIL", report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
tolog("LFC path: %s" % (lfcpath))
pat = re.compile("([^\.]+\.[^\.]+)\..*")
mat = pat.match(dsname)
if mat:
prefixdir = mat.group(1)
castor_destination = os.path.join(destination, prefixdir)
else:
pilotErrorDiag = "Unexpected dataset name format: %s" % (dsname)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport("DSN_FORMAT_FAIL", report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
tolog("SE destination: %s" % (castor_destination))
# set up paths differently for analysis and production jobs
# use conventional LFC paths or production jobs
# use OSG style for analysis jobs (for the time being)
if not analJob:
# return full lfc file path (beginning lfcpath might need to be replaced)
native_lfc_path = self.to_native_lfn(dsname, filename)
# /grid/atlas/dq2/testpanda/testpanda.destDB.b7cd4b56-1b5e-465a-a5d7-38d5e2609724_sub01000457/
# 58f836d5-ff4b-441a-979b-c37094257b72_0.job.log.tgz
tolog("Native_lfc_path: %s" % (native_lfc_path))
# replace the default path /grid/atlas/rucio with lfcpath if different
# (to_native_lfn returns a path begining with /grid/atlas/rucio)
default_lfcpath = "/grid/atlas/rucio" # to_native_lfn always returns this at the beginning of the string
if default_lfcpath != lfcpath:
final_lfc_path = native_lfc_path.replace(default_lfcpath, lfcpath)
else:
final_lfc_path = native_lfc_path
# name of dir to be created in LFC
lfcdir = os.path.dirname(final_lfc_path)
# /grid/atlas/dq2/testpanda/testpanda.destDB.dfb45803-1251-43bb-8e7a-6ad2b6f205be_sub01000492
tolog("LFC dir: %s" % (lfcdir))
# dst_gpfn = destination
# dst_gpfn = os.path.join(destination, os.path.join(dsname, filename))
# /pnfs/tier2.hep.manchester.ac.uk/data/atlas/dq2/testpanda/testpanda.destDB.dfb45803-1251-43bb-8e7a-6ad2b6f205be_sub01000492
# tolog("dst_gpfn: %s" % (dst_gpfn))
else: # for analysis jobs
lfcdir = "%s/%s/%s" % (lfcpath, prefixdir, dsname)
tolog("lfcdir: %s" % (lfcdir))
report["relativeStart"] = time()
# name of dir to be created on Castor
dirname = os.path.join(castor_destination, dsname)
dst_gpfn = os.path.join(castor_destination, os.path.join(dsname, filename))
tolog("dst_gpfn: %s" % (dst_gpfn))
fppfn = os.path.abspath(pfn)
# get the RSE from ToA
try:
_RSE = self.getRSE(surl=dst_gpfn)
except Exception, e:
tolog("Warning: Failed to get RSE: %s (can not add this info to tracing report)" % str(e))
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
"""Data transfer: includes BNL dir creation"""
error = PilotErrors()
pilotErrorDiag = ""
tolog("BNLdCacheSiteMover::put_data() called")
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
dsname = pdict.get('dsname', '')
logFile = pdict.get('logFile', '')
timeout = pdict.get('timeout', False)
analJob = pdict.get('analJob', False)
testLevel = pdict.get('testLevel', '0')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn,
guid)
ec, pilotErrorDiag = verifySetupCommand(error, self._setup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
filename = os.path.basename(source)
_tmp = destination.split('/', 3)
locse = '/' + _tmp[3]
ftpserv = _tmp[0] + '//' + _tmp[2]
subpath = self.genSubpath(dsname, filename, logFile)
sedir = os.path.join(locse, subpath)
m = re.search('^user', filename)
if m == None:
sedir = sedir + '/' + dsname
# create higher level log directory with 775 permission since jobs writting
# to here could be ran under usatlas1, usatlas2, ...
if sedir.find('/user_log02/testpanda/') != -1:
sedirh = os.path.dirname(sedir.rstrip('/'))
if not os.path.exists(sedirh):
cmd = 'mkdir -p -m 775 %s' % (sedirh)
# tolog("Executing command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
check_syserr(s, o)
pilotErrorDiag = "Error in mkdir: %s" % (o)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.prepareReport('MKDIR_FAIL', report)
return self.put_data_retfail(error.ERR_MKDIR,
pilotErrorDiag)
cmd = 'mkdir -p %s' % (sedir)
tolog("Executing command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
check_syserr(s, o)
pilotErrorDiag = "Error in mkdir: %s" % (o)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.prepareReport('MKDIR_FAIL', report)
return self.put_data_retfail(error.ERR_MKDIR, pilotErrorDiag)
report['validateStart'] = time.time()
tolog("Getting local file size")
try:
fsize = str(os.path.getsize(source))
except OSError, e:
pilotErrorDiag = "Could not get file size: %s" % str(e)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.prepareReport('NO_FILESIZE', report)
return self.put_data_retfail(error.ERR_FAILEDSIZELOCAL,
pilotErrorDiag)
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
"""
Moves the file from the current local directory to a storage element
source: full path of the file in local directory
destination: destination SE, method://[hostname[:port]]/full-dir-path/ (NB: no file name)
Assumes that the SE is locally mounted and its local path is the same as the remote path
if both fsize and fchecksum (for the source) are given and !=0 these are assumed without reevaluating them
returns: exitcode, gpfn,fsize, fchecksum
"""
error = PilotErrors()
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
dsname = pdict.get('dsname', '')
analJob = pdict.get('analJob', False)
extradirs = pdict.get('extradirs', '')
prodSourceLabel = pdict.get('prodSourceLabel', '')
if prodSourceLabel == 'ddm' and analJob:
tolog("Treating PanDA Mover job as a production job during stage-out")
analJob = False
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn, guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
report['relativeStart'] = time()
ec = 0
if fsize == 0 or fchecksum == 0:
if not self.useExternalAdler32():
# Can not use external adler32 command for remote file since the command is
# not available (defaulting to md5sum for put operation)
tolog("Command not found: adler32.sh (will switch to md5sum for local file checksum)")
csumtype = "default"
else:
csumtype = "adler32"
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(source, csumtype=csumtype)
if ec != 0:
self.__sendReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
tolog("File destination: %s" % (destination))
dst_se = destination
# srm://dcsrm.usatlas.bnl.gov:8443/srm/managerv1?SFN=/pnfs/usatlas.bnl.gov/
if( dst_se.find('SFN') != -1 ):
s = dst_se.split('SFN=')
dst_loc_se = s[1]
dst_prefix = s[0] + 'SFN='
else:
_sentries = dst_se.split('/', 3)
# 'method://host:port' is it always a ftp server? can it be srm? something else?
dst_serv = _sentries[0] + '//' + _sentries[2]
# dst_host = _sentries[2] # host and port
dst_loc_se = '/'+ _sentries[3]
dst_prefix = dst_serv
# use bare destination when it starts with root://
if destination.startswith('root://'):
dst_loc_se = destination
dst_prefix = ''
# report['dataset'] = dsname
# May be be a comma list but take first always
# (Remember that se can be a list where the first is used for output but any can be used for input)
se = readpar('se').split(",")[0]
_dummytoken, se = self.extractSE(se)
tolog("Using SE: %s" % (se))
filename = os.path.basename(source)
# are we transfering to a space token?
if token != None and token != "":
# get the proper destination
destination = self.getDestination(analJob, token)
if destination == '':
pilotErrorDiag = "put_data destination path in SE not defined"
tolog('!!WARNING!!2990!! %s' % (pilotErrorDiag))
self.__sendReport('SE_DEST_PATH_UNDEF', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
tolog("Going to store job output at destination: %s" % (destination))
# add the space token to the destination string
dst_loc_sedir = os.path.join(destination, os.path.join(extradirs, dsname))
dst_loc_pfn = os.path.join(dst_loc_sedir, filename)
dst_loc_pfn += "?oss.cgroup=%s" % (token)
else:
dst_loc_sedir = os.path.join(dst_loc_se, os.path.join(extradirs, dsname))
dst_loc_pfn = os.path.join(dst_loc_sedir, filename)
dst_gpfn = dst_prefix + dst_loc_pfn
tolog("Final destination path: %s" % (dst_loc_pfn))
tolog("dst_gpfn: %s" % (dst_gpfn))
# get the DQ2 site name from ToA
try:
_dq2SiteName = self.getDQ2SiteName(surl=dst_gpfn)
except Exception, e:
tolog("Warning: Failed to get the DQ2 site name: %s (can not add this info to tracing report)" % str(e))
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" stage-in function """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
experiment = pdict.get('experiment', '')
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'castor', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# get the experiment object
thisExperiment = getExperiment(experiment)
# do we have a valid proxy?
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup)
if s != 0:
self.prepareReport('PROXYFAIL', report)
return s, pilotErrorDiag
# Strip off prefix in order to use rfcp directly
tolog("gpfn: %s" % (gpfn))
pat = re.compile('^.*(/castor/.*)$')
mat = pat.match(gpfn)
if mat:
getfile = mat.group(1)
else:
pilotErrorDiag = "Get file not in castor: %s" % (gpfn)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport('NO_FILE', report)
return error.ERR_STAGEINFAILED, pilotErrorDiag
# when the file has been copied we will rename it to the lfn (to remove the legacy __DQ2-string on some files)
dest_path = os.path.join(path, lfn)
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# transfer the input file with rfcp
_cmd_str = '%srfcp %s %s' % (envsetup, getfile, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
s, o = commands.getstatusoutput(_cmd_str)
report['validateStart'] = time()
if s != 0:
o = o.replace('\n', ' ')
check_syserr(s, o)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog(
"!!WARNING!!1112!! Failed to remove local file, get retry will fail"
)
if o.find("No such file or directory") >= 0:
if getfile.find("DBRelease") >= 0:
pilotErrorDiag = "Missing DBRelease file: %s" % (getfile)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
ec = error.ERR_MISSDBREL
else:
pilotErrorDiag = "No such file or directory: %s" % (
getfile)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
ec = error.ERR_NOSUCHFILE
else:
pilotErrorDiag = "rfcp failed: %d, %s" % (s, o)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
ec = error.ERR_STAGEINFAILED
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# check file size and checksum
if fsize != 0 or fchecksum != 0:
# which checksum type are we using?
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# get remote file size and checksum
ec, pilotErrorDiag, dstfsize, dstfchecksum = self.getLocalFileInfo(
dest_path, csumtype=csumtype)
tolog("File info: %d, %s, %s" % (ec, dstfsize, dstfchecksum))
if ec != 0:
self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog(
"!!WARNING!!1112!! Failed to remove local file, get retry will fail"
)
return ec, pilotErrorDiag
# compare remote and local file size
if fsize != 0 and dstfsize != fsize:
pilotErrorDiag = "Remote and local file sizes do not match for %s (%s != %s)" %\
(os.path.basename(gpfn), str(dstfsize), str(fsize))
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.prepareReport('FS_MISMATCH', report)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog(
"!!WARNING!!1112!! Failed to remove local file, get retry will fail"
)
return error.ERR_GETWRONGSIZE, pilotErrorDiag
# compare remote and local file checksum
if fchecksum != 0 and dstfchecksum != fchecksum and not self.isDummyChecksum(
fchecksum):
pilotErrorDiag = "Remote and local checksums (of type %s) do not match for %s (%s != %s)" %\
(csumtype, os.path.basename(gpfn), dstfchecksum, fchecksum)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog(
"!!WARNING!!1112!! Failed to remove local file, get retry will fail"
)
if csumtype == "adler32":
self.prepareReport('AD_MISMATCH', report)
return error.ERR_GETADMISMATCH, pilotErrorDiag
else:
self.prepareReport('MD5_MISMATCH', report)
return error.ERR_GETMD5MISMATCH, pilotErrorDiag
updateFileState(lfn,
workDir,
jobId,
mode="file_state",
state="transferred",
ftype="input")
self.prepareReport('DONE', report)
return 0, pilotErrorDiag
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
token = pdict.get('token', None)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
proxycheck = pdict.get('proxycheck', False)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
if proxycheck:
# do we have a valid proxy?
s, pilotErrorDiag = self.verifyProxy(envsetup=envsetup)
if s != 0:
self.__sendReport('PROXYFAIL', report)
return s, pilotErrorDiag
else:
tolog("Proxy verification turned off")
getfile = gpfn
if path == '': path = './'
fullname = os.path.join(path, lfn)
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# get remote filesize and checksum
if fsize == 0 or fchecksum == 0:
try:
import lfc
except Exception, e:
pilotErrorDiag = "get_data() could not import lfc module: %s" % str(e)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.__sendReport('LFC_IMPORT', report)
return error.ERR_GETLFCIMPORT, pilotErrorDiag
os.environ['LFC_HOST'] = readpar('lfchost')
try:
ret, res = lfc.lfc_getreplicas([str(guid)],"")
except Exception, e:
pilotErrorDiag = "Failed to get LFC replicas: %s" % str(e)
tolog("!!WARNING!!2990!! Exception caught: %s" % (pilotErrorDiag))
tolog("Mover get_data finished (failed)")
self.__sendReport('NO_LFC_REPS', report)
return error.ERR_FAILEDLFCGETREPS, pilotErrorDiag
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
proxycheck = pdict.get('proxycheck', False)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg2', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
if proxycheck:
# do we have a valid proxy?
s, pilotErrorDiag = self.verifyProxy(envsetup=envsetup)
if s != 0:
self.__sendReport('PROXYFAIL', report)
return s, pilotErrorDiag
else:
tolog("Proxy verification turned off")
getfile = gpfn
if path == '': path = './'
fullname = os.path.join(path, lfn)
# can not test filesize and checksum if remote values are not known
if fsize == 0 or fchecksum == 0:
tolog("!!WARNING!!2999!! Remote file size/checksum not known: %s/%s" % (fsize, fchecksum))
# Maybe be a comma list but take first always
# (Remember that se can be a list where the first is used for output but any can be used for input)
se = readpar('se').split(",")[0]
_dummytoken, se = self.extractSE(se)
tolog("Using SE: %s" % (se))
# se = srm://head01.aglt2.org:8443/srm/managerv2?SFN=
# for srm protocol, use the full info from 'se'
if getfile[:3] == "srm":
try:
# e.g. tmp = ['srm:', '', 'head01.aglt2.org', 'pnfs/aglt2.org/dq2/panda/dis/08/...']
tmp = getfile.split('/',3)[2]
except Exception, e:
tolog('!!WARNING!!2999!! Could not extract srm protocol for replacement, keeping getfile variable as it is: %s (%s)' %\
(getfile, str(e)))
else:
# replace srm with 'srm://head01.aglt2.org:8443/srm/managerv2?SFN=' if not there already
if not '?SFN=' in getfile:
# srm = 'srm://head01.aglt2.org'
srm = 'srm://' + tmp
# does seopt contain any matching srm's?
sematch = self.getSEMatchFromSEOpt(srm)
if sematch != "":
getfile = getfile.replace(srm, sematch)
tolog("Replaced %s with %s (from seopt) in getfile: %s" % (srm, sematch, getfile))
else:
getfile = getfile.replace(srm, se)
tolog("Replaced %s with %s (from se) in getfile: %s" % (srm, se, getfile))
else:
tolog("Found SFN part in getfile: %s" % (getfile))
# add port number from se to getfile if necessary
getfile = self.addPortToPath(se, getfile)
return SiteMover.put_data_retfail(error.ERR_FAILEDSIZE,
pilotErrorDiag,
surl=dst_gpfn)
if fsize != nufsize:
pilotErrorDiag = "File sizes do not match for %s (%s != %s)" %\
(os.path.basename(source), str(fsize), str(nufsize))
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.prepareReport('FS_MISMATCH', report)
return SiteMover.put_data_retfail(error.ERR_PUTWRONGSIZE,
pilotErrorDiag,
surl=dst_gpfn)
# get a proper envsetup
envsetup = self.getEnvsetup()
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag, surl=dst_gpfn)
# register the file in LFC
report['catStart'] = time()
os.environ['LFC_HOST'] = readpar(
'lfchost') # might already be set in envsetup
lfcpath = readpar('lfcpath')
lfclfn = '%s/%s/%s/%s' % (lfcpath, prefixdir, dsname, lfn)
cmd = "%slcg-rf -g %s -l %s --vo atlas %s" % (envsetup, guid, lfclfn,
putfile)
tolog("registration command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
""" copy output file from disk to local SE """
# function is based on dCacheSiteMover put function
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
alt = pdict.get('alt', False)
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
scope = pdict.get('scope', '')
dsname = pdict.get('dsname', '')
analysisJob = pdict.get('analJob', False)
testLevel = pdict.get('testLevel', '0')
extradirs = pdict.get('extradirs', '')
experiment = pdict.get('experiment', '')
proxycheck = pdict.get('proxycheck', False)
prodSourceLabel = pdict.get('prodSourceLabel', '')
# get the site information object
si = getSiteInformation(experiment)
if prodSourceLabel == 'ddm' and analysisJob:
tolog(
"Treating PanDA Mover job as a production job during stage-out"
)
analysisJob = False
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg2', lfn, guid)
# preparing variables
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(
source, csumtype="adler32")
if ec != 0:
self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
# get the checksum type
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# get a proper envsetup
if alt:
# use a cvmfs setup for stage-out to alternative SE
envsetup = si.getLocalEMISetup()
if envsetup[-1] != ";":
envsetup += "; "
else:
envsetup = self.getEnvsetup(alt=alt)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# get the experiment object
thisExperiment = getExperiment(experiment)
if proxycheck:
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup,
limit=2)
if s != 0:
self.prepareReport('NO_PROXY', report)
return self.put_data_retfail(error.ERR_NOPROXY, pilotErrorDiag)
else:
tolog("Proxy verification turned off")
filename = os.path.basename(source)
# get all the proper paths
ec, pilotErrorDiag, tracer_error, dst_gpfn, lfcdir, surl = si.getProperPaths(
error,
analysisJob,
token,
prodSourceLabel,
dsname,
filename,
scope=scope,
alt=alt,
sitemover=self) # quick workaround
if ec != 0:
self.prepareReport(tracer_error, report)
return self.put_data_retfail(ec, pilotErrorDiag, surl=dst_gpfn)
putfile = surl
full_surl = putfile
if full_surl[:len('token:')] == 'token:':
# remove the space token (e.g. at Taiwan-LCG2) from the SURL info
full_surl = full_surl[full_surl.index('srm://'):]
# srm://dcache01.tier2.hep.manchester.ac.uk/pnfs/tier2.hep.manchester.ac.uk/data/atlas/dq2/
#testpanda.destDB/testpanda.destDB.604b4fbc-dbe9-4b05-96bb-6beee0b99dee_sub0974647/
#86ecb30d-7baa-49a8-9128-107cbfe4dd90_0.job.log.tgz
tolog("putfile = %s" % (putfile))
tolog("full_surl = %s" % (full_surl))
# get the RSE from ToA
try:
_RSE = self.getRSE(surl=putfile)
except Exception, e:
tolog(
"Warning: Failed to get RSE: %s (can not add this info to tracing report)"
% str(e))
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
""" copy output file from disk to local SE """
# function is based on dCacheSiteMover put function
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
alt = pdict.get('alt', False)
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
dsname = pdict.get('dsname', '')
analysisJob = pdict.get('analJob', False)
testLevel = pdict.get('testLevel', '0')
extradirs = pdict.get('extradirs', '')
experiment = pdict.get('experiment', '')
proxycheck = pdict.get('proxycheck', False)
prodSourceLabel = pdict.get('prodSourceLabel', '')
# get the site information object
si = getSiteInformation(experiment)
tolog("put_data received prodSourceLabel=%s" % (prodSourceLabel))
if prodSourceLabel == 'ddm' and analysisJob:
tolog("Treating PanDA Mover job as a production job during stage-out")
analysisJob = False
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg2', lfn, guid)
# preparing variables
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(source, csumtype="adler32")
if ec != 0:
self.__sendReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
# get the checksum type
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# get a proper envsetup
envsetup = self.getEnvsetup(alt=alt)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
if proxycheck:
s, pilotErrorDiag = self.verifyProxy(envsetup=envsetup, limit=2)
if s != 0:
self.__sendReport('NO_PROXY', report)
return self.put_data_retfail(error.ERR_NOPROXY, pilotErrorDiag)
else:
tolog("Proxy verification turned off")
filename = os.path.basename(source)
# get all the proper paths
ec, pilotErrorDiag, tracer_error, dst_gpfn, lfcdir, surl = si.getProperPaths(error, analysisJob, token, prodSourceLabel, dsname, filename, alt=alt)
if ec != 0:
self.__sendReport(tracer_error, report)
return self.put_data_retfail(ec, pilotErrorDiag)
putfile = surl
full_surl = putfile
if full_surl[:len('token:')] == 'token:':
# remove the space token (e.g. at Taiwan-LCG2) from the SURL info
full_surl = full_surl[full_surl.index('srm://'):]
# srm://dcache01.tier2.hep.manchester.ac.uk/pnfs/tier2.hep.manchester.ac.uk/data/atlas/dq2/
#testpanda.destDB/testpanda.destDB.604b4fbc-dbe9-4b05-96bb-6beee0b99dee_sub0974647/
#86ecb30d-7baa-49a8-9128-107cbfe4dd90_0.job.log.tgz
tolog("putfile = %s" % (putfile))
tolog("full_surl = %s" % (full_surl))
# get the DQ2 site name from ToA
try:
_dq2SiteName = self.getDQ2SiteName(surl=putfile)
except Exception, e:
tolog("Warning: Failed to get the DQ2 site name: %s (can not add this info to tracing report)" % str(e))
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
experiment = pdict.get('experiment', '')
proxycheck = pdict.get('proxycheck', False)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg2', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# get the experiment object
thisExperiment = getExperiment(experiment)
if proxycheck:
# do we have a valid proxy?
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup)
if s != 0:
self.prepareReport('PROXYFAIL', report)
return s, pilotErrorDiag
else:
tolog("Proxy verification turned off")
getfile = gpfn
if path == '': path = './'
fullname = os.path.join(path, lfn)
# can not test filesize and checksum if remote values are not known
if fsize == 0 or fchecksum == 0:
tolog(
"!!WARNING!!2999!! Remote file size/checksum not known: %s/%s"
% (fsize, fchecksum))
# Maybe be a comma list but take first always
# (Remember that se can be a list where the first is used for output but any can be used for input)
se = readpar('se').split(",")[0]
_dummytoken, se = self.extractSE(se)
tolog("Using SE: %s" % (se))
# se = srm://head01.aglt2.org:8443/srm/managerv2?SFN=
# for srm protocol, use the full info from 'se'
if getfile[:3] == "srm":
try:
# e.g. tmp = ['srm:', '', 'head01.aglt2.org', 'pnfs/aglt2.org/rucio/panda/dis/08/...']
tmp = getfile.split('/', 3)[2]
except Exception, e:
tolog('!!WARNING!!2999!! Could not extract srm protocol for replacement, keeping getfile variable as it is: %s (%s)' %\
(getfile, str(e)))
else:
# replace srm with 'srm://head01.aglt2.org:8443/srm/managerv2?SFN=' if not there already
if not '?SFN=' in getfile:
# srm = 'srm://head01.aglt2.org'
srm = 'srm://' + tmp
# does seopt contain any matching srm's?
sematch = self.getSEMatchFromSEOpt(srm)
if sematch != "":
getfile = getfile.replace(srm, sematch)
tolog(
"Replaced %s with %s (from seopt) in getfile: %s" %
(srm, sematch, getfile))
else:
getfile = getfile.replace(srm, se)
tolog("Replaced %s with %s (from se) in getfile: %s" %
(srm, se, getfile))
else:
tolog("Found SFN part in getfile: %s" % (getfile))
# add port number from se to getfile if necessary
getfile = self.addPortToPath(se, getfile)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'local', lfn, guid)
if not path:
tolog('path is empty, using current directory')
path = os.getcwd()
# build setup string
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(self.__error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return self.__error.ERR_DIRECTIOFILE, self.__pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("not directIn")
# build the get command
_params = ""
if fsize != 0 and fsize != "0":
_params += self.__par_filesize % (fsize,)
if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
csumtype = self.getChecksumType(fchecksum)
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
csumtype = 'md5'
_params += self.__par_checksum % ("%s:%s" % (csumtype, fchecksum),)
# add the guid option
_params += " --guid %s" % (guid)
dest_path = os.path.join(path, lfn)
#PN
# if ".lib." in gpfn:
# localGet = self.__localget
# else:
# localGet = self.__localgetBAD
# execStr = localGet % (envsetup, _params, gpfn, dest_path)
execStr = self.__localget % (envsetup, _params, gpfn, dest_path)
tolog("Executing command: %s" % (execStr))
report['transferStart'] = time()
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
""" copy output file from local dir to SE and register into dataset and catalogues """
# Get input parameters from pdict
# Mancinelli: added sitename and appid variable
sitename = pdict.get('sitename', '')
appid = pdict.get('report').get('appid','')
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
dsname = pdict.get('dsname', '')
workDir = pdict.get('workDir', '')
analyJob = pdict.get('analJob', False)
extradirs = pdict.get('extradirs', '')
experiment = pdict.get('experiment', '')
prodSourceLabel = pdict.get('prodSourceLabel', '')
# get the site information object
si = getSiteInformation(experiment)
if prodSourceLabel == 'ddm' and analyJob:
tolog("Treating PanDA Mover job as a production job during stage-out")
analyJob = False
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'local'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
report['guid'] = guid.replace('-','')
# report['dataset'] = dsname
filename = os.path.basename(source)
# get the local file size and checksum
csumtype = self.checksum_command
if fsize == 0 or fchecksum == 0:
ec, self.__pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(source, csumtype=csumtype)
if ec != 0:
self.__sendReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, self.__pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
# get a proper envsetup
envsetup = self.getEnvsetup()
ec, pilotErrorDiag = verifySetupCommand(self.__error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
#Mancinelli: TODO change. This is a Hack.. need to undrestand how to get Job data in a proper manner
#JobData= '%s/Job_%s.py' % (os.path.dirname(source), appid)
JobData= '%s/jobState-%s-test.pickle' % (os.path.dirname(source), appid)
# get all the proper paths
ec, pilotErrorDiag, tracer_error, dst_gpfn, lfcdir, surl = si.getProperPaths(self.__error, analyJob, token, prodSourceLabel, dsname, filename, sitename, JobData)
if ec != 0:
self.__sendReport(tracer_error, report)
return self.put_data_retfail(ec, pilotErrorDiag)
dst_gpfn = surl
tolog("dst_gpfn: %s" % (dst_gpfn))
# get the DQ2 site name from ToA
try:
_dq2SiteName = self.getDQ2SiteName(surl=dst_gpfn)
except Exception, e:
tolog("Warning: Failed to get the DQ2 site name: %s (can not add this info to tracing report)" % str(e))
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'rfcpLFC'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-','')
tolog("gpfn is %s" % gpfn)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
try:
if not loc_pfn.startswith(('/dpm', '/castor')):
tolog("Potential problem with local filename. Does not start with '/dpm' or '/castor/'.")
except TypeError:
# Older version of python
pass
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
#PN
_cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# if ".lib." in loc_pfn:
# _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# else:
# _cmd_str = '%srfcpXXX %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout
except OSError, e:
pilotErrorDiag = "put_data could not get file size: %s" % str(e)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.__sendReport('NO_FS', report)
return SiteMover.put_data_retfail(error.ERR_FAILEDSIZE, pilotErrorDiag)
if fsize != nufsize:
pilotErrorDiag = "File sizes do not match for %s (%s != %s)" %\
(os.path.basename(source), str(fsize), str(nufsize))
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.__sendReport('FS_MISMATCH', report)
return SiteMover.put_data_retfail(error.ERR_PUTWRONGSIZE, pilotErrorDiag, surl=dst_gpfn)
# get a proper envsetup
envsetup = self.getEnvsetup()
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# register the file in LFC
report['catStart'] = time()
os.environ['LFC_HOST'] = readpar('lfchost') # might already be set in envsetup
lfcpath = readpar('lfcpath')
lfclfn = '%s/%s/%s/%s' % (lfcpath, prefixdir, dsname, lfn)
cmd = "%slcg-rf -g %s -l %s --vo atlas %s" % (envsetup, guid, lfclfn, putfile)
tolog("registration command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
o = o.replace('\n', ' ')
pilotErrorDiag = "LFC registration failed: %s" % (o)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
Moves a DS file the local SE (where was put from DDM) to the working directory.
Performs the copy and, for systems supporting it, checks size and md5sum correctness
gpfn: full source URL (e.g. method://[host[:port]/full-dir-path/filename - a SRM URL is OK)
path: destination absolute path (in a local file system)
returns the status of the transfer. In case of failure it should remove the partially copied destination
"""
# The local file is assumed to have a relative path that is the same of the relative path in the 'gpfn'
# loc_... are the variables used to access the file in the locally exported file system
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
sitename = pdict.get('sitename', '')
cmtconfig = pdict.get('cmtconfig', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'xrdcp', lfn, guid)
# get a proper setup
_setup_str = self.getSetup()
# if "CERN" in sitename:
# _setup_str = "source /afs/cern.ch/project/xrootd/software/setup_stable_for_atlas.sh;"
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
src_loc_pfn = self.extractPathFromPFN(gpfn)
# determine which copy command to use
cpt = "xrdcp"
# Do pre-stagin
copytoolin = readpar('copytoolin')
# copytoolin = "xrdcp" #PN readpar('copytoolin')
tolog("xrdcpSiteMover ELN copytoolin : %s" % (copytoolin))
if copytoolin != '':
if copytoolin.find('^') > -1:
cpt, pstage = copytoolin.split('^')
if pstage != "" and pstage != 'dummy':
# xrdcp is anyway hardcoded below...
cmd = "%s %s " % (pstage, src_loc_pfn)
rc, rs, pilotErrorDiag = self.copy(cmd, stagein=True)
if rc != 0:
self.__sendReport('PSTAGE_FAIL', report)
return rc, pilotErrorDiag
else:
tolog("Successfully pre-staged file")
else:
cpt = readpar('copytoolin')
#cpt = "xrdcp" #PN readpar('copytoolin')
# is there a special copytool to be used?
# if cmtconfig != "" and "CERN" in sitename:
# # special path at CERN since default command is broken (as of November 2011)
# if "x86_64" in cmtconfig:
# # export LD_LIBRARY_PATH:/afs/cern.ch/sw/lcg/external/xrootd/3.1.0/x86_64-slc5-gcc43-opt/lib64/:/afs/cern.ch/sw/lcg/contrib/gcc/4.3.3/x86_64-slc5-gcc43-opt/lib64/
# lib_path1 = "/afs/cern.ch/sw/lcg/external/xrootd/3.1.0/%s/lib64" % (cmtconfig)
# lib_path2 = "/afs/cern.ch/sw/lcg/contrib/gcc/4.3.3/%s/lib64" % (cmtconfig)
# else:
# lib_path1 = "/afs/cern.ch/sw/lcg/external/xrootd/3.1.0/%s/lib" % (cmtconfig)
# lib_path2 = "/afs/cern.ch/sw/lcg/contrib/gcc/4.3.3/%s/lib" % (cmtconfig)
# command_path = "/afs/cern.ch/sw/lcg/external/xrootd/3.1.0/%s/bin/xrdcp" % (cmtconfig)
# status, badPath = self.verifyPaths([lib_path1, lib_path2, command_path])
# if status:
# cpt = "export LD_LIBRARY_PATH=%s:%s:$LD_LIBRARY_PATH;" % (lib_path1, lib_path2)
# cpt += command_path
# else:
# tolog("Path %s does not exist" % (badPath))
# pilotErrorDiag = "Failed to figure out a proper path for xrdcp: %s" % (badPath)
# tolog("!!WARNING!!1776!! %s" % (pilotErrorDiag))
# self.__sendReport('PSTAGE_FAIL', report)
# return error.ERR_STAGEINFAILED, pilotErrorDiag
# if "x86_64" in cmtconfig and "slc5" in cmtconfig:
# _path = "/afs/cern.ch/project/xrootd/software/setup_stable_for_atlas_slc5_x86_64.sh"
# elif "x86_64" in cmtconfig and "slc6" in cmtconfig:
# _path = "/afs/cern.ch/project/xrootd/software/setup_stable_for_atlas_slc6_x86_64.sh"
# else:
# _path = "/afs/cern.ch/project/xrootd/software/setup_stable_for_atlas_slc5_i686.sh"
# status, badPath = self.verifyPaths([_path])
# if status:
# cpt = "source %s; xrdcp" % (path)
tolog("Site mover will use get command: %s" % (cpt))
# copyprefixin = 'dcap://ccdcapatlas.in2p3.fr:22125^root://ccsrb15:1094'
# gpfn = 'srm://ccsrm.in2p3.fr/pnfs/in2p3.fr/data/...'
# src_loc_pfn = '/pnfs/in2p3.fr/data/atlas/...'
# add 'root://ccsrb15:1094' to src_loc_pfn
copyprefix = readpar('copyprefixin')
if copyprefix == "":
copyprefix = readpar('copyprefix')
tolog("Using copyprefix = %s" % (copyprefix))
else:
tolog("Using copyprefixin = %s" % (copyprefix))
if copyprefix == "":
pilotErrorDiag = "Empty copyprefix, cannot continue"
tolog("!!WARNING!!1777!! %s" % (pilotErrorDiag))
self.__sendReport('PSTAGE_FAIL', report)
return error.ERR_STAGEINFAILED, pilotErrorDiag
# handle copyprefix lists
pfroms, ptos = getCopyprefixLists(copyprefix)
if len(pfroms) != len(ptos):
pilotErrorDiag = "Copyprefix lists not of equal length: %s, %s" % (str(pfroms), str(ptos))
tolog("!!WARNING!!1777!! %s" % (pilotErrorDiag))
self.__sendReport('PSTAGE_FAIL', report)
return error.ERR_STAGEINFAILED, pilotErrorDiag
for (pfrom, pto) in map(None, pfroms, ptos):
if (pfrom != "" and pfrom != None and pfrom != "dummy") and (pto != "" and pto != None and pto != "dummy"):
if gpfn[:len(pfrom)] == pfrom or gpfn[:len(pto)] == pto:
src_loc_pfn = pto + src_loc_pfn
src_loc_pfn = src_loc_pfn.replace('///','//')
break
tolog("PFN=%s" % (gpfn))
tolog("TURL=%s" % (src_loc_pfn))
src_loc_filename = lfn # os.path.basename(src_loc_pfn)
# source vars: gpfn, loc_pfn, loc_host, loc_dirname, loc_filename
# dest vars: path
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# protect against bad pfn's
# src_loc_pfn = src_loc_pfn.replace('///','/')
src_loc_pfn = src_loc_pfn.replace('//xrootd/','/xrootd/')
if src_loc_pfn.find('//pnfs') == -1:
src_loc_pfn = src_loc_pfn.replace('/pnfs','//pnfs')
# should the root file be copied or read directly by athena?
directIn = self.checkForDirectAccess(lfn, useCT, workDir, jobId, prodDBlockToken)
if directIn:
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('FOUND_ROOT', report)
return error.ERR_DIRECTIOFILE, pilotErrorDiag
# in case fchecksum is not given to this function, attempt to use the md5 option to get it
dest_file = os.path.join(path, src_loc_filename)
if fchecksum == 0 or fchecksum == "" or fchecksum == None or fchecksum == "None":
useMd5Option = True
cmd = "%s %s -md5 %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
else:
useMd5Option = False
cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
# is the md5 option available?
if useMd5Option:
cmd_test = "%s %s" % (_setup_str, cpt)
tolog("Executing test command: %s" % (cmd_test))
rc, rs = commands.getstatusoutput(cmd_test)
if rs.find("-md5") > 0:
tolog("This xrdcp version supports the md5 option")
else:
tolog("This xrdcp version does not support the md5 option (checksum test will be skipped)")
useMd5Option = False
cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
#PN
# if not ".lib." in src_loc_pfn:
# cmd = "xrd-cpXXX -h"
# add the full stage-out command to the job setup script
to_script = cmd.replace(dest_file, "`pwd`/%s" % os.path.basename(dest_file))
to_script = to_script.lstrip(' ') # remove any initial spaces
if to_script.startswith('/'):
to_script = 'source ' + to_script
addToJobSetupScript(to_script, path)
# transfer the file
report['transferStart'] = time()
rc, rs, pilotErrorDiag = self.copy(cmd, stagein=True)
report['validateStart'] = time()
if rc != 0:
self.__sendReport('COPY_FAIL', report)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_file)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
return rc, pilotErrorDiag
else:
tolog("Successfully transferred file")
# get file size from the command output if not known already
if fsize == 0:
fsize = self.getFileSize(rs)
# get checksum from the command output if not known already
if useMd5Option and fchecksum == 0:
fchecksum = self.getChecksum(rs)
else:
if fchecksum == 0 or fchecksum == None:
fchecksum = ""
else:
tolog("fchecksum = %s" % (fchecksum))
# get destination (local) file size and checksum
ec, pilotErrorDiag, dstfsize, dstfchecksum = self.getLocalFileInfo(dest_file, csumtype=csumtype)
tolog("File info: %d, %s, %s" % (ec, dstfsize, dstfchecksum))
if ec != 0:
self.__sendReport('LOCAL_FILE_INFO_FAIL', report)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_file)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
return ec, pilotErrorDiag
# compare remote and local file checksum
if fchecksum != "" and fchecksum != 0 and dstfchecksum != fchecksum and not self.isDummyChecksum(fchecksum):
pilotErrorDiag = "Remote and local checksums (of type %s) do not match for %s (%s != %s)" %\
(csumtype, os.path.basename(gpfn), fchecksum, dstfchecksum)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_file)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
if csumtype == "adler32":
self.__sendReport('AD_MISMATCH', report)
return error.ERR_GETADMISMATCH, pilotErrorDiag
else:
self.__sendReport('MD5_MISMATCH', report)
return error.ERR_GETMD5MISMATCH, pilotErrorDiag
# compare remote and local file size (skip test if remote/source file size is not known)
if dstfsize != fsize and fsize != 0 and fsize != "":
pilotErrorDiag = "Remote and local file sizes do not match for %s (%s != %s)" %\
(os.path.basename(gpfn), str(dstfsize), str(fsize))
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.__sendReport('FS_MISMATCH', report)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_file)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
return error.ERR_GETWRONGSIZE, pilotErrorDiag
updateFileState(lfn, workDir, jobId, mode="file_state", state="transferred", type="input")
self.__sendReport('DONE', report)
return 0, pilotErrorDiag
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
"""Data transfer: includes BNL dir creation"""
error = PilotErrors()
pilotErrorDiag = ""
tolog("BNLdCacheSiteMover::put_data() called")
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
dsname = pdict.get('dsname', '')
logFile = pdict.get('logFile', '')
timeout = pdict.get('timeout', False)
analJob = pdict.get('analJob', False)
testLevel = pdict.get('testLevel', '0')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn, guid)
ec, pilotErrorDiag = verifySetupCommand(error, self._setup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
filename = os.path.basename(source)
_tmp = destination.split('/', 3)
locse = '/'+ _tmp[3]
ftpserv = _tmp[0] + '//' + _tmp[2]
subpath = self.genSubpath(dsname, filename, logFile)
sedir = os.path.join(locse, subpath)
m = re.search('^user', filename)
if m == None:
sedir = sedir + '/' + dsname
# create higher level log directory with 775 permission since jobs writting
# to here could be ran under usatlas1, usatlas2, ...
if sedir.find('/user_log02/testpanda/') != -1:
sedirh = os.path.dirname(sedir.rstrip('/'))
if not os.path.exists(sedirh):
cmd = 'mkdir -p -m 775 %s' % (sedirh)
# tolog("Executing command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
check_syserr(s, o)
pilotErrorDiag = "Error in mkdir: %s" % (o)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.__sendReport('MKDIR_FAIL', report)
return self.put_data_retfail(error.ERR_MKDIR, pilotErrorDiag)
cmd = 'mkdir -p %s' % (sedir)
tolog("Executing command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
check_syserr(s, o)
pilotErrorDiag = "Error in mkdir: %s" % (o)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.__sendReport('MKDIR_FAIL', report)
return self.put_data_retfail(error.ERR_MKDIR, pilotErrorDiag)
report['validateStart'] = time.time()
tolog("Getting local file size")
try:
fsize = str(os.path.getsize(source))
except OSError, e:
pilotErrorDiag = "Could not get file size: %s" % str(e)
tolog('!!WARNING!!2999!! %s' % (pilotErrorDiag))
self.__sendReport('NO_FILESIZE', report)
return self.put_data_retfail(error.ERR_FAILEDSIZELOCAL, pilotErrorDiag)
def put_data(self,
pfn,
ddm_storage,
fsize=0,
fchecksum=0,
dsname='',
extradirs='',
**pdict):
""" Copy all output file to the local SE """
error = PilotErrors()
pilotErrorDiag = ""
tolog("put_data() got ddm_storage=%s" % (ddm_storage))
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
analJob = pdict.get('analJob', False)
experiment = pdict.get('experiment', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'castor', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup()
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# get the experiment object
thisExperiment = getExperiment(experiment)
# do we have a valid proxy?
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup,
limit=2)
if s != 0:
self.prepareReport('PROXYFAIL', report)
return self.put_data_retfail(s, pilotErrorDiag)
filename = pfn.split('/')[-1]
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-', '')
# Destination is the top level Castor store area. Append a subdirectory which is first two fields of dsname, or 'other'
destination = ""
if not analJob:
# seprodpath can have a complex structure in case of space tokens
# although currently not supported in this site mover, prepare the code anyway
# (use the first list item only)
destination = self.getDirList(readpar('seprodpath'))[0]
if destination == "":
tolog("!!WARNING!!2999!! seprodpath not defined, using sepath")
destination = readpar('sepath')
tolog("Going to store production job output")
else:
destination = readpar('sepath')
tolog("Going to store analysis job output")
if destination == '':
pilotErrorDiag = "put_data destination path in SE not defined"
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport('DEST_PATH_UNDEF', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED,
pilotErrorDiag)
else:
tolog("destination: %s" % (destination))
if dsname == '':
pilotErrorDiag = "Dataset name not specified to put_data"
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport('NO_DSN', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED,
pilotErrorDiag)
# else:
# dsname = self.remove_sub(dsname)
# tolog("dsname: %s" % (dsname))
lfcpath, pilotErrorDiag = self.getLFCPath(analJob)
if lfcpath == "":
self.prepareReport('LFC_PATH_FAIL', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED,
pilotErrorDiag)
tolog("LFC path: %s" % (lfcpath))
pat = re.compile('([^\.]+\.[^\.]+)\..*')
mat = pat.match(dsname)
if mat:
prefixdir = mat.group(1)
castor_destination = os.path.join(destination, prefixdir)
else:
pilotErrorDiag = "Unexpected dataset name format: %s" % (dsname)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport('DSN_FORMAT_FAIL', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED,
pilotErrorDiag)
tolog("SE destination: %s" % (castor_destination))
# set up paths differently for analysis and production jobs
# use conventional LFC paths or production jobs
# use OSG style for analysis jobs (for the time being)
if not analJob:
# return full lfc file path (beginning lfcpath might need to be replaced)
native_lfc_path = self.to_native_lfn(dsname, filename)
# /grid/atlas/dq2/testpanda/testpanda.destDB.b7cd4b56-1b5e-465a-a5d7-38d5e2609724_sub01000457/
#58f836d5-ff4b-441a-979b-c37094257b72_0.job.log.tgz
tolog("Native_lfc_path: %s" % (native_lfc_path))
# replace the default path /grid/atlas/rucio with lfcpath if different
# (to_native_lfn returns a path begining with /grid/atlas/rucio)
default_lfcpath = '/grid/atlas/rucio' # to_native_lfn always returns this at the beginning of the string
if default_lfcpath != lfcpath:
final_lfc_path = native_lfc_path.replace(
default_lfcpath, lfcpath)
else:
final_lfc_path = native_lfc_path
# name of dir to be created in LFC
lfcdir = os.path.dirname(final_lfc_path)
# /grid/atlas/dq2/testpanda/testpanda.destDB.dfb45803-1251-43bb-8e7a-6ad2b6f205be_sub01000492
tolog("LFC dir: %s" % (lfcdir))
# dst_gpfn = destination
# dst_gpfn = os.path.join(destination, os.path.join(dsname, filename))
# /pnfs/tier2.hep.manchester.ac.uk/data/atlas/dq2/testpanda/testpanda.destDB.dfb45803-1251-43bb-8e7a-6ad2b6f205be_sub01000492
# tolog("dst_gpfn: %s" % (dst_gpfn))
else: # for analysis jobs
lfcdir = '%s/%s/%s' % (lfcpath, prefixdir, dsname)
tolog("lfcdir: %s" % (lfcdir))
report['relativeStart'] = time()
# name of dir to be created on Castor
dirname = os.path.join(castor_destination, dsname)
dst_gpfn = os.path.join(castor_destination,
os.path.join(dsname, filename))
tolog("dst_gpfn: %s" % (dst_gpfn))
fppfn = os.path.abspath(pfn)
# get the RSE from ToA
try:
_RSE = self.getRSE(surl=dst_gpfn)
except Exception, e:
tolog(
"Warning: Failed to get RSE: %s (can not add this info to tracing report)"
% str(e))
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5*3600)
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time.time()
pnfsid = self.getPnfsid(loc_pfn, guid)
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if pnfsid == None:
isStaged = self.isFileStaged(_setup_str, loc_pfn)
else:
_com = "/cacheinfos/isFileInPool?pnfsid=%s" % (pnfsid)
isStaged = self.isFileStaged(_setup_str, loc_pfn, url="ddmv02.usatlas.bnl.gov:8000", com=_com)
if not isStaged:
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (loc_pfn)
self.__sendReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
if pnfsid == None:
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
else:
_cmd_str = '%sdccp pnfs://dcdcap.usatlas.bnl.gov:22125/%s %s' % (_setup_str, pnfsid, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time.time()
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
TODO: document GPFN format (SURL from catalog srm://host/path)
TODO: document better constraint
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'castorSVC', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
# Hard code the configuration dictionary for now, but eventually this should be
# set dynamically.
#
# There are the following configuration sections:
# setup - base environment veriables to be set
# svcClassMap - dictionary of string matches vs. service class names
# svcClassList - list of all service classes in case the svcClassMap matching fails
# svcClassDefault - the service class to set if the file appears to be staged no where
#
# Information from RAL:
# [root@srm0661 ~]# listStorageArea -v atlas
# <Space Token> <Description> <service class> <type> <status>
# 4948ef55-0000-1000-b7dd-9b38bdd87201 "ATLASGROUP" "atlasStripDeg" "DURABLE" "ALLOCATED"
# 4948ef38-0000-1000-8606-973e4e998e02 "ATLASMCDISK" "atlasSimStrip" "DURABLE" "ALLOCATED"
# 4948eec6-0000-1000-8ca2-aba0529b4806 "ATLASDATADISK" "atlasStripInput" "DURABLE" "ALLOCATED"
# 4948ee8e-0000-1000-9ac5-81bb9b34ba7b "ATLASMCTAPE" "atlasSimRaw" "PERMANENT" "ALLOCATED"
# 4948ee71-0000-1000-b611-a0afad31f6c8 "ATLASDATATAPE" "atlasT0Raw" "PERMANENT" "ALLOCATED"
# "ATLASHOTDISK" "atlasHotDisk"
# In addition there is the "atlasFarm" class, which is used when data is staged back from tape
castorConfig = {
'setup': {
'STAGE_HOST': 'catlasstager.ads.rl.ac.uk',
'STAGER_HOST': 'catlasstager.ads.rl.ac.uk',
'RFIO_USE_CASTOR_V2': 'YES',
},
'svcClassList': (
'atlasHotDisk',
'atlasSimStrip',
'atlasStripInput',
'atlasFarm',
'atlasStripDeg',
'atlasT0Raw',
'atlasSimRaw',
'atlasScratchDisk',
),
'svcClassMap': {
'/atlashotdisk/': 'atlasHotDisk',
'/atlasmcdisk/': 'atlasStripInput',
'/atlasdatadisk/': 'atlasStripInput',
'/atlasgroupdisk/': 'atlasStripDeg',
'/atlasdatatape/': 'atlasFarm',
'/atlasmctape/': 'atlasFarm',
'/atlasscratchdisk/': 'atlasScratchDisk',
'/atlasProdDisk/': 'atlasScratchDisk',
},
'svcClassDefault':
'atlasFarm',
}
# Set all environment variables for castor setup
for envVar, value in castorConfig['setup'].iteritems():
os.environ[envVar] = value
# Strip the gpfn (SURL) back to its bare castor component
tolog("gpfn is %s" % gpfn)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
loc_pfn = ''
if (gpfn.find('SFN') != -1):
s = gpfn.split('SFN=')
loc_pfn = s[1]
tolog("Found SFN string. Local file name %s" % loc_pfn)
else:
_tmp = gpfn.split('/', 3)
loc_pfn = '/' + _tmp[3]
tolog("Splitting SURL on slashes. Got local file name %s" %
loc_pfn)
if not loc_pfn.startswith('/castor/'):
tolog(
"WARNING: Problem with local filename: Does not start with '/castor/'."
)
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# Now need to find the service class associated with the file.
# If we find a clear indication of a space token in the file path
# then this is easy. However, if we don't, then use stager_qry to
# interrogate each possible service class. If this fails then use
# atlasFarm in desperation.
serviceClass = None
for pathMatch, svcClass in castorConfig['svcClassMap'].iteritems():
if loc_pfn.find(pathMatch) >= 0:
tolog('Matched path element %s - service class is %s' %
(pathMatch, svcClass))
serviceClass = svcClass
break
else:
tolog('Path element %s for service class %s - no match' %
(pathMatch, svcClass))
# For testing the fallback, then we need to hobble ourselves by unsetting serviceClass:
#tolog('Automatic service class was: %s' % serviceClass)
#tolog('Unsetting service class for fallback testing')
#serviceClass = None
if serviceClass == None:
tolog("Warning: Failed to find service class hint in SURL.")
for tryMe in castorConfig['svcClassList']:
os.environ['STAGE_SVCCLASS'] = tryMe
tolog('Trying service class %s for file' % tryMe)
err, output = commands.getstatusoutput('stager_qry -M %s' %
loc_pfn)
if err != 0:
tolog(
'WARNING: Unexpected status from stager_qry: %d\n%s' %
(err, output))
else:
if output.find('STAGED') >= 0:
tolog('Found file in service class %s' % tryMe)
serviceClass = tryMe
break
else:
tolog('File not found in service class %s' % tryMe)
if serviceClass == None:
tolog(
'WARNING: Failed to find file in any expected service class - will set STAGE_SVCCLASS to %s'
% castorConfig['svcClassDefault'])
serviceClass = castorConfig['svcClassDefault']
tolog('Setting STAGE_SVCCLASS to %s' % serviceClass)
os.environ['STAGE_SVCCLASS'] = serviceClass
dest_path = os.path.join(path, lfn)
_cmd_str = '%s/usr/bin/rfcp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
""" Moves the file from the current local directory to a storage element
source: full path of the file in local directory
destination: destination SE, method://[hostname[:port]]/full-dir-path/ (NB: no file name)
Assumes that the SE is locally mounted and its local path is the same as the remote path
if both fsize and fchecksum (for the source) are given and !=0 these are assumed without reevaluating them
returns: exitcode, gpfn,fsize, fchecksum
"""
error = PilotErrors()
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
scope = pdict.get('scope', '')
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
dsname = pdict.get('dsname', '')
analyJob = pdict.get('analyJob', False)
extradirs = pdict.get('extradirs', '')
experiment = pdict.get('experiment', '')
prodSourceLabel = pdict.get('prodSourceLabel', '')
# get the site information object
si = getSiteInformation(experiment)
if prodSourceLabel == 'ddm' and analyJob:
tolog(
"Treating PanDA Mover job as a production job during stage-out"
)
analyJob = False
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn,
guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
report['relativeStart'] = time()
ec = 0
if fsize == 0 or fchecksum == 0:
if not self.useExternalAdler32():
# Can not use external adler32 command for remote file since the command is
# not available (defaulting to md5sum for put operation)
tolog(
"Command not found: adler32.sh (will switch to md5sum for local file checksum)"
)
csumtype = "default"
else:
csumtype = "adler32"
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(
source, csumtype=csumtype)
if ec != 0:
self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
tolog("File destination: %s" % (destination))
dst_se = destination
# srm://dcsrm.usatlas.bnl.gov:8443/srm/managerv1?SFN=/pnfs/usatlas.bnl.gov/
if (dst_se.find('SFN') != -1):
s = dst_se.split('SFN=')
dst_loc_se = s[1]
dst_prefix = s[0] + 'SFN='
else:
_sentries = dst_se.split('/', 3)
# 'method://host:port' is it always a ftp server? can it be srm? something else?
dst_serv = _sentries[0] + '//' + _sentries[2]
# dst_host = _sentries[2] # host and port
dst_loc_se = '/' + _sentries[3]
dst_prefix = dst_serv
# use bare destination when it starts with root://
if destination.startswith('root://'):
dst_loc_se = destination
dst_prefix = ''
# report['dataset'] = dsname
# May be be a comma list but take first always
# (Remember that se can be a list where the first is used for output but any can be used for input)
se = readpar('se').split(",")[0]
_dummytoken, se = self.extractSE(se)
tolog("Using SE: %s" % (se))
filename = os.path.basename(source)
ec, pilotErrorDiag, tracer_error, dst_gpfn, lfcdir, surl = si.getProperPaths(
error,
analyJob,
token,
prodSourceLabel,
dsname,
filename,
scope=scope,
sitemover=self) # quick workaround
if ec != 0:
self.prepareReport(tracer_error, report)
return self.put_data_retfail(ec, pilotErrorDiag)
# are we transfering to a space token?
if token != None and token != "":
# Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places)
if "dst:" in token:
token = token[len('dst:'):]
tolog("Dropped dst: part of space token descriptor; token=%s" %
(token))
token = "ATLASGROUPDISK"
tolog("Space token descriptor reset to: %s" % (token))
# get the proper destination
#destination = self.getDestination(analyJob, token)
#if destination == '':
# pilotErrorDiag = "put_data destination path in SE not defined"
# tolog('!!WARNING!!2990!! %s' % (pilotErrorDiag))
# self.prepareReport('SE_DEST_PATH_UNDEF', report)
# return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
#tolog("Going to store job output at destination: %s" % (destination))
# add the space token to the destination string
#dst_loc_sedir = os.path.join(destination, os.path.join(extradirs, dsname))
#dst_loc_pfn = os.path.join(dst_loc_sedir, filename)
#dst_loc_pfn += "?oss.cgroup=%s" % (token)
dst_loc_pfn = dst_gpfn + "?oss.cgroup=%s" % (token)
#else:
#dst_loc_sedir = os.path.join(dst_loc_se, os.path.join(extradirs, dsname))
#dst_loc_pfn = os.path.join(dst_loc_sedir, filename)
dst_loc_pfn = dst_gpfn
dst_gpfn = dst_prefix + dst_loc_pfn
tolog("Final destination path: %s" % (dst_loc_pfn))
tolog("dst_gpfn: %s" % (dst_gpfn))
# get the Rucio site name from ToA
try:
_RSE = self.getRSE(surl=dst_gpfn)
except Exception, e:
tolog(
"Warning: Failed to get RSE: %s (can not add this info to tracing report)"
% str(e))
def put_data(self, pfn, destination, fsize=0, fchecksum=0, dsname='', extradirs='', **pdict):
""" copy output file from disk to local SE """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
logFile = pdict.get('logFile', '')
sitename = pdict.get('sitename', '')
proxycheck = pdict.get('proxycheck', False)
experiment = pdict.get('experiment', '')
analysisJob = pdict.get('analJob', False)
prodSourceLabel = pdict.get('prodSourceLabel', '')
# get the site information object
si = getSiteInformation(experiment)
if prodSourceLabel == 'ddm' and analysisJob:
tolog("Treating PanDA Mover job as a production job during stage-out")
analysisJob = False
filename = pfn.split('/')[-1]
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg', lfn, guid)
# is the dataset defined?
if dsname == '':
pilotErrorDiag = "Dataset name not specified to put_data"
tolog('!!WARNING!!2990!! %s' % (pilotErrorDiag))
self.__sendReport('DSN_UNDEF', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
# preparing variables
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(pfn, csumtype="adler32")
if ec != 0:
self.__sendReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
# get a proper envsetup
envsetup = self.getEnvsetup()
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# do we need to check the user proxy?
if proxycheck:
s, pilotErrorDiag = self.verifyProxy(envsetup=envsetup, limit=2)
if s != 0:
self.__sendReport('PROXY_FAIL', report)
return self.put_data_retfail(error.ERR_NOPROXY, pilotErrorDiag)
else:
tolog("Proxy verification turned off")
# get all the proper paths
ec, pilotErrorDiag, tracer_error, dst_gpfn, lfcdir, surl = si.getProperPaths(error, analysisJob, token, prodSourceLabel, dsname, filename)
if ec != 0:
self.__sendReport(tracer_error, report)
return self.put_data_retfail(ec, pilotErrorDiag)
lfclfn = os.path.join(lfcdir, lfn)
# LFC LFN = /grid/atlas/dq2/testpanda/testpanda.destDB.dfb45803-1251-43bb-8e7a-6ad2b6f205be_sub01000492/
#364aeb74-8b62-4c8f-af43-47b447192ced_0.job.log.tgz
# putfile is the SURL
putfile = surl
full_surl = putfile
if full_surl[:len('token:')] == 'token:':
# remove the space token (e.g. at Taiwan-LCG2) from the SURL info
full_surl = full_surl[full_surl.index('srm://'):]
# srm://dcache01.tier2.hep.manchester.ac.uk/pnfs/tier2.hep.manchester.ac.uk/data/atlas/dq2/
#testpanda.destDB/testpanda.destDB.604b4fbc-dbe9-4b05-96bb-6beee0b99dee_sub0974647/
#86ecb30d-7baa-49a8-9128-107cbfe4dd90_0.job.log.tgz
tolog("putfile = %s" % (putfile))
tolog("full_surl = %s" % (full_surl))
# get the DQ2 site name from ToA
try:
_dq2SiteName = self.getDQ2SiteName(surl=putfile)
except:
# WARNING: do not print the exception here since it can sometimes not be converted to a string! (problem seen at Taiwan)
tolog("Warning: Failed to get the DQ2 site name (can not add this info to tracing report)")
else:
report['localSite'], report['remoteSite'] = (_dq2SiteName, _dq2SiteName)
tolog("DQ2 site name: %s" % (_dq2SiteName))
# get the absolute (full) path to the file
fppfn = os.path.abspath(pfn)
tolog("pfn=%s" % (pfn))
cmd = '%s echo "LFC_HOST=$LFC_HOST"; lfc-mkdir -p %s' % (envsetup, lfcdir)
# export LFC_HOST=lfc0448.gridpp.rl.ac.uk ; echo "LFC_HOST=$LFC_HOST";
#lfc-mkdir -p /grid/atlas/dq2/testpanda.destDB/testpanda.destDB.604b4fbc-dbe9-4b05-96bb-6beee0b99dee_sub0974647
tolog("Executing command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s == 0:
tolog("LFC setup and mkdir succeeded")
tolog("Command output: %s" % (o))
else:
tolog("!!WARNING!!2990!! LFC setup and mkdir failed. Status=%s Output=%s" % (s, o))
if o == "Could not establish context":
pilotErrorDiag = "Could not establish context: Proxy / VO extension of proxy has probably expired"
tolog("!!WARNING!!2990!! %s" % (pilotErrorDiag))
self.dumpExtendedProxy(envsetup)
self.__sendReport('CONTEXT_FAIL', report)
return self.put_data_retfail(error.ERR_NOPROXY, pilotErrorDiag)
else:
pilotErrorDiag = "LFC setup and mkdir failed: %s" % (o)
self.__sendReport('LFC_SETUP_FAIL', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
# determine which timeout option to use
if self.isNewLCGVersion("%s lcg-cr" % (envsetup)):
timeout_option = "--srm-timeout=%d --connect-timeout=300 --sendreceive-timeout=%d" % (self.timeout, self.timeout)
else:
timeout_option = "-t %d" % (self.timeout)
# used lcg-cr options:
# --verbose: verbosity on
# --vo: specifies the Virtual Organization the user belongs to
# -T: specify SRM version
# -s: space token description
# -b: BDII disabling
# -t: time-out
# -l: specifies the Logical File Name associated with the file. If this option is present, an entry is added to the LFC
# -g: specifies the Grid Unique IDentifier. If this option is not present, a GUID is generated internally
# -d: specifies the destination. It can be the Storage Element fully qualified hostname or an SURL. In the latter case,
# the scheme can be sfn: for a classical SE or srm:. If only the fully qualified hostname is given, a filename is
# generated in the same format as with the Replica Manager
if token:
surl = putfile[putfile.index('srm://'):]
_cmd_str = '%s which lcg-cr; lcg-cr --version; lcg-cr --verbose --vo atlas -T srmv2 -s %s -b %s -l %s -g %s -d %s file:%s' % (envsetup, token, timeout_option, lfclfn, guid, surl, fppfn)
else:
surl = putfile
_cmd_str = '%s which lcg-cr; lcg-cr --version; lcg-cr --verbose --vo atlas %s -l %s -g %s -d %s file:%s' % (envsetup, timeout_option, lfclfn, guid, surl, fppfn)
tolog("Executing command: %s" % (_cmd_str))
s = -1
t0 = os.times()
report['relativeStart'] = time()
report['transferStart'] = time()
try:
s, o = commands.getstatusoutput(_cmd_str)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught: %s" % (str(e)))
o = str(e)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
Moves a DS file the local SE (where was put from DDM) to the working directory.
Performs the copy and, for systems supporting it, checks size and md5sum correctness
gpfn: full source URL (e.g. method://[host[:port]/full-dir-path/filename - a SRM URL is OK)
path: destination absolute path (in a local file system)
returns the status of the transfer. In case of failure it should remove the partially copied destination
"""
# The local file is assumed to have a relative path that is the same of the relative path in the 'gpfn'
# loc_... are the variables used to access the file in the locally exported file system
error = PilotErrors()
pilotErrorDiag = ""
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
dsname = pdict.get('dsname', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn,
guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
tolog("xrootdSiteMover get_data using setup: %s" % (_setup_str))
# remove any host and SFN info from PFN path
src_loc_pfn = self.extractPathFromPFN(gpfn)
src_loc_filename = lfn
# source vars: gpfn, loc_pfn, loc_host, loc_dirname, loc_filename
# dest vars: path
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# protect against bad pfn's
src_loc_pfn = src_loc_pfn.replace('///', '/')
src_loc_pfn = src_loc_pfn.replace('//xrootd/', '/xrootd/')
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
rootFile = self.isRootFile(src_loc_pfn, setup=_setup_str)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file: %s (will not be transferred in direct reading mode)"
% (src_loc_pfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('IS_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("No direct access mode")
ec = 0
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(
src_loc_pfn, csumtype=csumtype)
if ec != 0:
self.prepareReport('GET_LOCAL_FILE_INFO_FAIL', report)
return ec, pilotErrorDiag
dest_file = os.path.join(path, src_loc_filename)
report['relativeStart'] = time()
# determine which copy command to use
cpt = self.getCopytool(_setup_str)
report['transferStart'] = time()
cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
#PN
# if ".lib." in src_loc_pfn:
# cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
# else:
# cmd = "%s %sXXX %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
tolog("Executing command: %s" % (cmd))
# execute
timeout = 3600
try:
rc, telapsed, cout, cerr = timed_command(cmd, timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(
e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
rc = 1
rs = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5 * 3600)
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time.time()
pnfsid = self.getPnfsid(loc_pfn, guid)
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if pnfsid == None:
isStaged = self.isFileStaged(_setup_str, loc_pfn)
else:
_com = "/cacheinfos/isFileInPool?pnfsid=%s" % (pnfsid)
isStaged = self.isFileStaged(
_setup_str,
loc_pfn,
url="ddmv02.usatlas.bnl.gov:8000",
com=_com)
if not isStaged:
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (
loc_pfn)
self.prepareReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
if pnfsid == None:
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
else:
_cmd_str = '%sdccp pnfs://dcdcap.usatlas.bnl.gov:22125/%s %s' % (
_setup_str, pnfsid, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time.time()
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
str(e))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5 * 3600)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'dCache', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time()
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if not self.isFileStaged(_setup_str, loc_pfn):
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (
loc_pfn)
self.prepareReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
telapsed = 0
try:
# t0 = time()
# s, cout = commands.getstatusoutput(_cmd_str)
# cerr = ""
# telapsed = time() - t0
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
print "DEBUG: s = ", s, type(s)
print "DEBUG: telapsed = ", telapsed, type(telapsed)
print "DEBUG: cout = ", cout, type(cout)
print "DEBUG: cerr = ", cerr, type(cerr)
cout = self.filter_text(cout)
cerr = self.filter_text(cerr)
if not self.is_number(s):
s = 1
if not self.is_number(telapsed):
telapsed = 0
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
(e))
s = 1
o = self.filter_text(e)
telapsed = timeout
# write traceback info to stderr
import traceback
exc, msg, tb = sys.exc_info()
traceback.print_tb(tb)
_pilotErrorDiag = "Unexpected exception: %s" % (get_exc_plus())
tolog("!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag))
print "!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
Moves a DS file the local SE (where was put from DDM) to the working directory.
Performs the copy and, for systems supporting it, checks size and md5sum correctness
gpfn: full source URL (e.g. method://[host[:port]/full-dir-path/filename - a SRM URL is OK)
path: destination absolute path (in a local file system)
returns the status of the transfer. In case of failure it should remove the partially copied destination
"""
# The local file is assumed to have a relative path that is the same of the relative path in the 'gpfn'
# loc_... are the variables used to access the file in the locally exported file system
error = PilotErrors()
pilotErrorDiag = ""
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
dsname = pdict.get('dsname', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn, guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
tolog("xrootdSiteMover get_data using setup: %s" % (_setup_str))
# remove any host and SFN info from PFN path
src_loc_pfn = self.extractPathFromPFN(gpfn)
src_loc_filename = lfn
# source vars: gpfn, loc_pfn, loc_host, loc_dirname, loc_filename
# dest vars: path
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# protect against bad pfn's
src_loc_pfn = src_loc_pfn.replace('///','/')
src_loc_pfn = src_loc_pfn.replace('//xrootd/','/xrootd/')
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
rootFile = self.isRootFile(src_loc_pfn, setup=_setup_str)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file: %s (will not be transferred in direct reading mode)" % (src_loc_pfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('IS_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("No direct access mode")
ec = 0
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(src_loc_pfn, csumtype=csumtype)
if ec != 0:
self.__sendReport('GET_LOCAL_FILE_INFO_FAIL', report)
return ec, pilotErrorDiag
dest_file = os.path.join(path, src_loc_filename)
report['relativeStart'] = time()
# determine which copy command to use
cpt = self.getCopytool(_setup_str)
report['transferStart'] = time()
cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
#PN
# if ".lib." in src_loc_pfn:
# cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
# else:
# cmd = "%s %sXXX %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
tolog("Executing command: %s" % (cmd))
# execute
timeout = 3600
try:
rc, telapsed, cout, cerr = timed_command(cmd, timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
rc = 1
rs = str(e)
telapsed = timeout
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
""" Data transfer using dCache - generic version """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
timeout = pdict.get('timeout', 5 * 3600)
dsname = pdict.get('dsname', '')
extradirs = pdict.get('extradirs', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'dCache', lfn,
guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# preparing variables
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(
source, csumtype="adler32")
if ec != 0:
self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
csumtype = "adler32"
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
dst_se = destination
if (
dst_se.find('SFN') != -1
): # srm://dcsrm.usatlas.bnl.gov:8443/srm/managerv1?SFN=/pnfs/usatlas.bnl.gov/
s = dst_se.split('SFN=')
dst_loc_se = s[1]
dst_prefix = s[0] + 'SFN='
else:
_sentries = dst_se.split('/', 3)
dst_serv = _sentries[0] + '//' + _sentries[
2] # 'method://host:port' is it always a ftp server? can it be srm? something else?
# dst_host = _sentries[2] # host and port
dst_loc_se = '/' + _sentries[3]
dst_prefix = dst_serv
filename = os.path.basename(source)
# report['dataset'] = dsname
# Behavior as in BNL: user files have no dsname automatically added to dir name
m = re.search('^user', filename)
if m:
dsname = ''
dst_loc_sedir = os.path.join(dst_loc_se,
os.path.join(extradirs, dsname))
dst_loc_pfn = os.path.join(dst_loc_sedir, filename)
dst_gpfn = dst_prefix + dst_loc_pfn
# get the RSE from ToA
try:
_RSE = self.getRSE(surl=dst_gpfn)
except Exception, e:
tolog(
"Warning: Failed to get RSE: %s (can not add this info to tracing report)"
% str(e))
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
""" Data transfer using dCache - generic version """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
timeout = pdict.get('timeout', 5*3600)
dsname = pdict.get('dsname', '')
extradirs = pdict.get('extradirs', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'dCache', lfn, guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# preparing variables
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(source, csumtype="adler32")
if ec != 0:
self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
csumtype = "adler32"
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
dst_se = destination
if( dst_se.find('SFN') != -1 ): # srm://dcsrm.usatlas.bnl.gov:8443/srm/managerv1?SFN=/pnfs/usatlas.bnl.gov/
s = dst_se.split('SFN=')
dst_loc_se = s[1]
dst_prefix = s[0] + 'SFN='
else:
_sentries = dst_se.split('/', 3)
dst_serv = _sentries[0] + '//' + _sentries[2] # 'method://host:port' is it always a ftp server? can it be srm? something else?
# dst_host = _sentries[2] # host and port
dst_loc_se = '/' + _sentries[3]
dst_prefix = dst_serv
filename = os.path.basename(source)
# report['dataset'] = dsname
# Behavior as in BNL: user files have no dsname automatically added to dir name
m = re.search('^user', filename)
if m:
dsname = ''
dst_loc_sedir = os.path.join(dst_loc_se, os.path.join(extradirs, dsname))
dst_loc_pfn = os.path.join(dst_loc_sedir, filename)
dst_gpfn = dst_prefix + dst_loc_pfn
# get the RSE from ToA
try:
_RSE = self.getRSE(surl=dst_gpfn)
except Exception, e:
tolog("Warning: Failed to get RSE: %s (can not add this info to tracing report)" % str(e))
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
token = pdict.get('token', None)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
experiment = pdict.get('experiment', '')
proxycheck = pdict.get('proxycheck', False)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# get the experiment object
thisExperiment = getExperiment(experiment)
if proxycheck:
# do we have a valid proxy?
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup)
if s != 0:
self.prepareReport('PROXYFAIL', report)
return s, pilotErrorDiag
else:
tolog("Proxy verification turned off")
getfile = gpfn
if path == '': path = './'
fullname = os.path.join(path, lfn)
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# get remote filesize and checksum
if fsize == 0 or fchecksum == 0:
try:
import lfc
except Exception, e:
pilotErrorDiag = "get_data() could not import lfc module: %s" % str(e)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport('LFC_IMPORT', report)
return error.ERR_GETLFCIMPORT, pilotErrorDiag
os.environ['LFC_HOST'] = readpar('lfchost')
try:
ret, res = lfc.lfc_getreplicas([str(guid)],"")
except Exception, e:
pilotErrorDiag = "Failed to get LFC replicas: %s" % str(e)
tolog("!!WARNING!!2990!! Exception caught: %s" % (pilotErrorDiag))
tolog("Mover get_data finished (failed)")
self.prepareReport('NO_LFC_REPS', report)
return error.ERR_FAILEDLFCGETREPS, pilotErrorDiag
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" stage-in function """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get("usect", True)
jobId = pdict.get("jobId", "")
workDir = pdict.get("workDir", "")
experiment = pdict.get("experiment", "")
prodDBlockToken = pdict.get("access", "")
# get the Rucio tracing report
report = self.getStubTracingReport(pdict["report"], "castor", lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport("RFCP_FAIL", report)
return ec, pilotErrorDiag
# get the experiment object
thisExperiment = getExperiment(experiment)
# do we have a valid proxy?
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup)
if s != 0:
self.prepareReport("PROXYFAIL", report)
return s, pilotErrorDiag
# Strip off prefix in order to use rfcp directly
tolog("gpfn: %s" % (gpfn))
pat = re.compile("^.*(/castor/.*)$")
mat = pat.match(gpfn)
if mat:
getfile = mat.group(1)
else:
pilotErrorDiag = "Get file not in castor: %s" % (gpfn)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport("NO_FILE", report)
return error.ERR_STAGEINFAILED, pilotErrorDiag
# when the file has been copied we will rename it to the lfn (to remove the legacy __DQ2-string on some files)
dest_path = os.path.join(path, lfn)
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == "local" or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn)
)
report["relativeStart"] = None
report["transferStart"] = None
self.prepareReport("FOUND_ROOT", report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", ftype="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# transfer the input file with rfcp
_cmd_str = "%srfcp %s %s" % (envsetup, getfile, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report["transferStart"] = time()
s, o = commands.getstatusoutput(_cmd_str)
report["validateStart"] = time()
if s != 0:
o = o.replace("\n", " ")
check_syserr(s, o)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
if o.find("No such file or directory") >= 0:
if getfile.find("DBRelease") >= 0:
pilotErrorDiag = "Missing DBRelease file: %s" % (getfile)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
ec = error.ERR_MISSDBREL
else:
pilotErrorDiag = "No such file or directory: %s" % (getfile)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
ec = error.ERR_NOSUCHFILE
else:
pilotErrorDiag = "rfcp failed: %d, %s" % (s, o)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
ec = error.ERR_STAGEINFAILED
self.prepareReport("RFCP_FAIL", report)
return ec, pilotErrorDiag
# check file size and checksum
if fsize != 0 or fchecksum != 0:
# which checksum type are we using?
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# get remote file size and checksum
ec, pilotErrorDiag, dstfsize, dstfchecksum = self.getLocalFileInfo(dest_path, csumtype=csumtype)
tolog("File info: %d, %s, %s" % (ec, dstfsize, dstfchecksum))
if ec != 0:
self.prepareReport("LOCAL_FILE_INFO_FAIL", report)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
return ec, pilotErrorDiag
# compare remote and local file size
if fsize != 0 and dstfsize != fsize:
pilotErrorDiag = "Remote and local file sizes do not match for %s (%s != %s)" % (
os.path.basename(gpfn),
str(dstfsize),
str(fsize),
)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport("FS_MISMATCH", report)
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
return error.ERR_GETWRONGSIZE, pilotErrorDiag
# compare remote and local file checksum
if fchecksum != 0 and dstfchecksum != fchecksum and not self.isDummyChecksum(fchecksum):
pilotErrorDiag = "Remote and local checksums (of type %s) do not match for %s (%s != %s)" % (
csumtype,
os.path.basename(gpfn),
dstfchecksum,
fchecksum,
)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
# remove the local file before any get retry is attempted
_status = self.removeLocal(dest_path)
if not _status:
tolog("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
if csumtype == "adler32":
self.prepareReport("AD_MISMATCH", report)
return error.ERR_GETADMISMATCH, pilotErrorDiag
else:
self.prepareReport("MD5_MISMATCH", report)
return error.ERR_GETMD5MISMATCH, pilotErrorDiag
updateFileState(lfn, workDir, jobId, mode="file_state", state="transferred", ftype="input")
self.prepareReport("DONE", report)
return 0, pilotErrorDiag
def put_data(self, pfn, destination, fsize=0, fchecksum=0, dsname='', extradirs='', **pdict):
""" copy output file from disk to local SE """
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
scope = pdict.get('scope', '')
logFile = pdict.get('logFile', '')
sitename = pdict.get('sitename', '')
proxycheck = pdict.get('proxycheck', False)
experiment = pdict.get('experiment', '')
analysisJob = pdict.get('analJob', False)
prodSourceLabel = pdict.get('prodSourceLabel', '')
# get the site information object
si = getSiteInformation(experiment)
if prodSourceLabel == 'ddm' and analysisJob:
tolog("Treating PanDA Mover job as a production job during stage-out")
analysisJob = False
filename = pfn.split('/')[-1]
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'lcg', lfn, guid)
# is the dataset defined?
if dsname == '':
pilotErrorDiag = "Dataset name not specified to put_data"
tolog('!!WARNING!!2990!! %s' % (pilotErrorDiag))
self.prepareReport('DSN_UNDEF', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag)
# preparing variables
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(pfn, csumtype="adler32")
if ec != 0:
self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
# get a proper envsetup
envsetup = self.getEnvsetup()
ec, pilotErrorDiag = verifySetupCommand(error, envsetup)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return self.put_data_retfail(ec, pilotErrorDiag)
# get the experiment object
thisExperiment = getExperiment(experiment)
# do we need to check the user proxy?
if proxycheck:
s, pilotErrorDiag = thisExperiment.verifyProxy(envsetup=envsetup, limit=2)
if s != 0:
self.prepareReport('PROXY_FAIL', report)
return self.put_data_retfail(error.ERR_NOPROXY, pilotErrorDiag)
else:
tolog("Proxy verification turned off")
# get all the proper paths
ec, pilotErrorDiag, tracer_error, dst_gpfn, lfcdir, surl = si.getProperPaths(error, analysisJob, token, prodSourceLabel, dsname, filename, scope=scope, sitemover=self) # quick workaround
if ec != 0:
self.prepareReport(tracer_error, report)
return self.put_data_retfail(ec, pilotErrorDiag, surl=dst_gpfn)
lfclfn = os.path.join(lfcdir, lfn)
# LFC LFN = /grid/atlas/dq2/testpanda/testpanda.destDB.dfb45803-1251-43bb-8e7a-6ad2b6f205be_sub01000492/
#364aeb74-8b62-4c8f-af43-47b447192ced_0.job.log.tgz
# putfile is the SURL
putfile = surl
full_surl = putfile
if full_surl[:len('token:')] == 'token:':
# remove the space token (e.g. at Taiwan-LCG2) from the SURL info
full_surl = full_surl[full_surl.index('srm://'):]
# srm://dcache01.tier2.hep.manchester.ac.uk/pnfs/tier2.hep.manchester.ac.uk/data/atlas/dq2/
#testpanda.destDB/testpanda.destDB.604b4fbc-dbe9-4b05-96bb-6beee0b99dee_sub0974647/
#86ecb30d-7baa-49a8-9128-107cbfe4dd90_0.job.log.tgz
tolog("putfile = %s" % (putfile))
tolog("full_surl = %s" % (full_surl))
# get the DQ2 site name from ToA
try:
_dq2SiteName = self.getDQ2SiteName(surl=putfile)
except:
# WARNING: do not print the exception here since it can sometimes not be converted to a string! (problem seen at Taiwan)
tolog("Warning: Failed to get the DQ2 site name (can not add this info to tracing report)")
else:
report['localSite'], report['remoteSite'] = (_dq2SiteName, _dq2SiteName)
tolog("DQ2 site name: %s" % (_dq2SiteName))
# get the absolute (full) path to the file
fppfn = os.path.abspath(pfn)
tolog("pfn=%s" % (pfn))
cmd = '%s echo "LFC_HOST=$LFC_HOST"; lfc-mkdir -p %s' % (envsetup, lfcdir)
# export LFC_HOST=lfc0448.gridpp.rl.ac.uk ; echo "LFC_HOST=$LFC_HOST";
#lfc-mkdir -p /grid/atlas/dq2/testpanda.destDB/testpanda.destDB.604b4fbc-dbe9-4b05-96bb-6beee0b99dee_sub0974647
tolog("Executing command: %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s == 0:
tolog("LFC setup and mkdir succeeded")
tolog("Command output: %s" % (o))
else:
tolog("!!WARNING!!2990!! LFC setup and mkdir failed. Status=%s Output=%s" % (s, o))
if o == "Could not establish context":
pilotErrorDiag = "Could not establish context: Proxy / VO extension of proxy has probably expired"
tolog("!!WARNING!!2990!! %s" % (pilotErrorDiag))
self.dumpExtendedProxy(envsetup)
self.prepareReport('CONTEXT_FAIL', report)
return self.put_data_retfail(error.ERR_NOPROXY, pilotErrorDiag, surl=full_surl)
else:
pilotErrorDiag = "LFC setup and mkdir failed: %s" % (o)
self.prepareReport('LFC_SETUP_FAIL', report)
return self.put_data_retfail(error.ERR_STAGEOUTFAILED, pilotErrorDiag, surl=full_surl)
# determine which timeout option to use
if self.isNewLCGVersion("%s lcg-cr" % (envsetup)):
timeout_option = "--srm-timeout=%d --connect-timeout=300 --sendreceive-timeout=%d" % (self.timeout, self.timeout)
else:
timeout_option = "-t %d" % (self.timeout)
# used lcg-cr options:
# --verbose: verbosity on
# --vo: specifies the Virtual Organization the user belongs to
# -T: specify SRM version
# -s: space token description
# -b: BDII disabling
# -t: time-out
# -l: specifies the Logical File Name associated with the file. If this option is present, an entry is added to the LFC
# -g: specifies the Grid Unique IDentifier. If this option is not present, a GUID is generated internally
# -d: specifies the destination. It can be the Storage Element fully qualified hostname or an SURL. In the latter case,
# the scheme can be sfn: for a classical SE or srm:. If only the fully qualified hostname is given, a filename is
# generated in the same format as with the Replica Manager
if token:
# Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places)
if "dst:" in token:
token = token[len('dst:'):]
tolog("Dropped dst: part of space token descriptor; token=%s" % (token))
token = "ATLASGROUPDISK"
tolog("Space token descriptor reset to: %s" % (token))
surl = putfile[putfile.index('srm://'):]
_cmd_str = '%s which lcg-cr; lcg-cr --version; lcg-cr --verbose --vo atlas -T srmv2 -s %s -b %s -l %s -g %s -d %s file:%s' % (envsetup, token, timeout_option, lfclfn, guid, surl, fppfn)
else:
surl = putfile
_cmd_str = '%s which lcg-cr; lcg-cr --version; lcg-cr --verbose --vo atlas %s -l %s -g %s -d %s file:%s' % (envsetup, timeout_option, lfclfn, guid, surl, fppfn)
# GoeGrid testing: _cmd_str = '%s which lcg-cr; lcg-cr --version; lcg-crXXX --verbose --vo atlas %s -l %s -g %s -d %s file:%s' % (envsetup, timeout_option, lfclfn, guid, surl, fppfn)
tolog("Executing command: %s" % (_cmd_str))
s = -1
t0 = os.times()
report['relativeStart'] = time()
report['transferStart'] = time()
try:
s, o = commands.getstatusoutput(_cmd_str)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught: %s" % (str(e)))
o = str(e)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
TODO: document GPFN format (SURL from catalog srm://host/path)
TODO: document better constraint
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'castorSVC', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
# Hard code the configuration dictionary for now, but eventually this should be
# set dynamically.
#
# There are the following configuration sections:
# setup - base environment veriables to be set
# svcClassMap - dictionary of string matches vs. service class names
# svcClassList - list of all service classes in case the svcClassMap matching fails
# svcClassDefault - the service class to set if the file appears to be staged no where
#
# Information from RAL:
# [root@srm0661 ~]# listStorageArea -v atlas
# <Space Token> <Description> <service class> <type> <status>
# 4948ef55-0000-1000-b7dd-9b38bdd87201 "ATLASGROUP" "atlasStripDeg" "DURABLE" "ALLOCATED"
# 4948ef38-0000-1000-8606-973e4e998e02 "ATLASMCDISK" "atlasSimStrip" "DURABLE" "ALLOCATED"
# 4948eec6-0000-1000-8ca2-aba0529b4806 "ATLASDATADISK" "atlasStripInput" "DURABLE" "ALLOCATED"
# 4948ee8e-0000-1000-9ac5-81bb9b34ba7b "ATLASMCTAPE" "atlasSimRaw" "PERMANENT" "ALLOCATED"
# 4948ee71-0000-1000-b611-a0afad31f6c8 "ATLASDATATAPE" "atlasT0Raw" "PERMANENT" "ALLOCATED"
# "ATLASHOTDISK" "atlasHotDisk"
# In addition there is the "atlasFarm" class, which is used when data is staged back from tape
castorConfig = {
'setup' : {
'STAGE_HOST' : 'catlasstager.ads.rl.ac.uk',
'STAGER_HOST' : 'catlasstager.ads.rl.ac.uk',
'RFIO_USE_CASTOR_V2' : 'YES',
},
'svcClassList' : ('atlasHotDisk', 'atlasSimStrip', 'atlasStripInput', 'atlasFarm', 'atlasStripDeg', 'atlasT0Raw', 'atlasSimRaw', 'atlasScratchDisk', ),
'svcClassMap' : {
'/atlashotdisk/' : 'atlasHotDisk',
'/atlasmcdisk/' : 'atlasStripInput',
'/atlasdatadisk/' : 'atlasStripInput',
'/atlasgroupdisk/' : 'atlasStripDeg',
'/atlasdatatape/' : 'atlasFarm',
'/atlasmctape/' : 'atlasFarm',
'/atlasscratchdisk/' : 'atlasScratchDisk',
'/atlasProdDisk/' : 'atlasScratchDisk',
},
'svcClassDefault' : 'atlasFarm',
}
# Set all environment variables for castor setup
for envVar, value in castorConfig['setup'].iteritems():
os.environ[envVar] = value
# Strip the gpfn (SURL) back to its bare castor component
tolog("gpfn is %s" % gpfn)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
loc_pfn = ''
if( gpfn.find('SFN') != -1 ):
s = gpfn.split('SFN=')
loc_pfn = s[1]
tolog("Found SFN string. Local file name %s" % loc_pfn)
else:
_tmp = gpfn.split('/', 3)
loc_pfn = '/'+_tmp[3]
tolog("Splitting SURL on slashes. Got local file name %s" % loc_pfn)
if not loc_pfn.startswith('/castor/'):
tolog("WARNING: Problem with local filename: Does not start with '/castor/'.")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", ftype="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# Now need to find the service class associated with the file.
# If we find a clear indication of a space token in the file path
# then this is easy. However, if we don't, then use stager_qry to
# interrogate each possible service class. If this fails then use
# atlasFarm in desperation.
serviceClass = None
for pathMatch, svcClass in castorConfig['svcClassMap'].iteritems():
if loc_pfn.find(pathMatch) >= 0:
tolog('Matched path element %s - service class is %s' % (pathMatch, svcClass))
serviceClass = svcClass
break
else:
tolog('Path element %s for service class %s - no match' % (pathMatch, svcClass))
# For testing the fallback, then we need to hobble ourselves by unsetting serviceClass:
#tolog('Automatic service class was: %s' % serviceClass)
#tolog('Unsetting service class for fallback testing')
#serviceClass = None
if serviceClass == None:
tolog("Warning: Failed to find service class hint in SURL.")
for tryMe in castorConfig['svcClassList']:
os.environ['STAGE_SVCCLASS'] = tryMe
tolog('Trying service class %s for file' % tryMe)
err, output = commands.getstatusoutput('stager_qry -M %s' % loc_pfn)
if err != 0:
tolog('WARNING: Unexpected status from stager_qry: %d\n%s' % (err, output))
else:
if output.find('STAGED') >= 0:
tolog('Found file in service class %s' % tryMe)
serviceClass = tryMe
break
else:
tolog('File not found in service class %s' % tryMe)
if serviceClass == None:
tolog('WARNING: Failed to find file in any expected service class - will set STAGE_SVCCLASS to %s' % castorConfig['svcClassDefault'])
serviceClass = castorConfig['svcClassDefault']
tolog('Setting STAGE_SVCCLASS to %s' % serviceClass)
os.environ['STAGE_SVCCLASS'] = serviceClass
dest_path = os.path.join(path, lfn)
_cmd_str = '%s/usr/bin/rfcp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'rfcpLFC'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-', '')
tolog("gpfn is %s" % gpfn)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
try:
if not loc_pfn.startswith(('/dpm', '/castor')):
tolog(
"Potential problem with local filename. Does not start with '/dpm' or '/castor/'."
)
except TypeError:
# Older version of python
pass
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
type="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
type="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
#PN
_cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# if ".lib." in loc_pfn:
# _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# else:
# _cmd_str = '%srfcpXXX %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout
