def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
TODO: document GPFN format
TODO: document better constraint
"""
error = PilotErrors()
pilotErrorDiag = ""
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'HU'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-','')
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
try:
timeout = pdict['timeout']
except:
timeout = 5*3600
if( gpfn.find('SFN') != -1 ):
s = gpfn.split('SFN=')
loc_pfn = s[1]
else:
_tmp = gpfn.split('/', 3)
loc_pfn = '/'+_tmp[3]
_cmd_str = '%snecp %s %s/%s' % (_setup_str, loc_pfn, path, lfn)
tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
report['transferStart'] = time()
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
TODO: document GPFN format
TODO: document better constraint
"""
error = PilotErrors()
pilotErrorDiag = ""
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'HU'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-','')
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
try:
timeout = pdict['timeout']
except:
timeout = 5*3600
if( gpfn.find('SFN') != -1 ):
s = gpfn.split('SFN=')
loc_pfn = s[1]
else:
_tmp = gpfn.split('/', 3)
loc_pfn = '/'+_tmp[3]
_cmd_str = '%snecp %s %s/%s' % (_setup_str, loc_pfn, path, lfn)
tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
report['transferStart'] = time()
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
s = 1
o = str(e)
telapsed = timeout
def copy(self, cmd, stagein=False):
""" Perfom the actual file transfer """
ec = 0
pilotErrorDiag = ""
error = PilotErrors()
tolog("Executing command: %s" % (cmd))
try:
s, telapsed, cout, cerr = timed_command(cmd, self.timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
s = 1
o = str(e)
telapsed = self.timeout
def check_space(self, storage_endpoint):
""" available spave in the SE """
# build setup string
envsetup = self.getEnvsetup()
# build the df command
execStr = self.__localspace % (envsetup, '', storage_endpoint)
tolog("Executing command: %s" % (execStr))
# execute
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout
except Exception, e:
tolog(
"Warning: Failed to get RSE: %s (can not add this info to tracing report)"
% str(e))
else:
report['localSite'], report['remoteSite'] = (_RSE, _RSE)
tolog("RSE: %s" % (_RSE))
if testLevel == "1":
source = "thisisjustatest"
cmd = 'source %s; dccp %s %s' % (self._setup, source, sedir)
tolog("Executing command: %s" % (cmd))
report['transferStart'] = time.time()
try:
s, telapsed, cout, cerr = timed_command(cmd, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
str(e))
s = 1
e = sys.exc_info()
o = str(e[1]) # set error message from exception text
telapsed = timeout
else:
o = cout + cerr
tolog("Elapsed time: %d" % (telapsed))
if s != 0:
o = o.replace('\n', ' ')
check_syserr(s, o)
pilotErrorDiag = "Error in copying: %s" % str(o)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
TODO: document GPFN format (SURL from catalog srm://host/path)
TODO: document better constraint
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'castorSVC', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
# Hard code the configuration dictionary for now, but eventually this should be
# set dynamically.
#
# There are the following configuration sections:
# setup - base environment veriables to be set
# svcClassMap - dictionary of string matches vs. service class names
# svcClassList - list of all service classes in case the svcClassMap matching fails
# svcClassDefault - the service class to set if the file appears to be staged no where
#
# Information from RAL:
# [root@srm0661 ~]# listStorageArea -v atlas
# <Space Token> <Description> <service class> <type> <status>
# 4948ef55-0000-1000-b7dd-9b38bdd87201 "ATLASGROUP" "atlasStripDeg" "DURABLE" "ALLOCATED"
# 4948ef38-0000-1000-8606-973e4e998e02 "ATLASMCDISK" "atlasSimStrip" "DURABLE" "ALLOCATED"
# 4948eec6-0000-1000-8ca2-aba0529b4806 "ATLASDATADISK" "atlasStripInput" "DURABLE" "ALLOCATED"
# 4948ee8e-0000-1000-9ac5-81bb9b34ba7b "ATLASMCTAPE" "atlasSimRaw" "PERMANENT" "ALLOCATED"
# 4948ee71-0000-1000-b611-a0afad31f6c8 "ATLASDATATAPE" "atlasT0Raw" "PERMANENT" "ALLOCATED"
# "ATLASHOTDISK" "atlasHotDisk"
# In addition there is the "atlasFarm" class, which is used when data is staged back from tape
castorConfig = {
'setup': {
'STAGE_HOST': 'catlasstager.ads.rl.ac.uk',
'STAGER_HOST': 'catlasstager.ads.rl.ac.uk',
'RFIO_USE_CASTOR_V2': 'YES',
},
'svcClassList': (
'atlasHotDisk',
'atlasSimStrip',
'atlasStripInput',
'atlasFarm',
'atlasStripDeg',
'atlasT0Raw',
'atlasSimRaw',
'atlasScratchDisk',
),
'svcClassMap': {
'/atlashotdisk/': 'atlasHotDisk',
'/atlasmcdisk/': 'atlasStripInput',
'/atlasdatadisk/': 'atlasStripInput',
'/atlasgroupdisk/': 'atlasStripDeg',
'/atlasdatatape/': 'atlasFarm',
'/atlasmctape/': 'atlasFarm',
'/atlasscratchdisk/': 'atlasScratchDisk',
'/atlasProdDisk/': 'atlasScratchDisk',
},
'svcClassDefault':
'atlasFarm',
}
# Set all environment variables for castor setup
for envVar, value in castorConfig['setup'].iteritems():
os.environ[envVar] = value
# Strip the gpfn (SURL) back to its bare castor component
tolog("gpfn is %s" % gpfn)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
loc_pfn = ''
if (gpfn.find('SFN') != -1):
s = gpfn.split('SFN=')
loc_pfn = s[1]
tolog("Found SFN string. Local file name %s" % loc_pfn)
else:
_tmp = gpfn.split('/', 3)
loc_pfn = '/' + _tmp[3]
tolog("Splitting SURL on slashes. Got local file name %s" %
loc_pfn)
if not loc_pfn.startswith('/castor/'):
tolog(
"WARNING: Problem with local filename: Does not start with '/castor/'."
)
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# Now need to find the service class associated with the file.
# If we find a clear indication of a space token in the file path
# then this is easy. However, if we don't, then use stager_qry to
# interrogate each possible service class. If this fails then use
# atlasFarm in desperation.
serviceClass = None
for pathMatch, svcClass in castorConfig['svcClassMap'].iteritems():
if loc_pfn.find(pathMatch) >= 0:
tolog('Matched path element %s - service class is %s' %
(pathMatch, svcClass))
serviceClass = svcClass
break
else:
tolog('Path element %s for service class %s - no match' %
(pathMatch, svcClass))
# For testing the fallback, then we need to hobble ourselves by unsetting serviceClass:
#tolog('Automatic service class was: %s' % serviceClass)
#tolog('Unsetting service class for fallback testing')
#serviceClass = None
if serviceClass == None:
tolog("Warning: Failed to find service class hint in SURL.")
for tryMe in castorConfig['svcClassList']:
os.environ['STAGE_SVCCLASS'] = tryMe
tolog('Trying service class %s for file' % tryMe)
err, output = commands.getstatusoutput('stager_qry -M %s' %
loc_pfn)
if err != 0:
tolog(
'WARNING: Unexpected status from stager_qry: %d\n%s' %
(err, output))
else:
if output.find('STAGED') >= 0:
tolog('Found file in service class %s' % tryMe)
serviceClass = tryMe
break
else:
tolog('File not found in service class %s' % tryMe)
if serviceClass == None:
tolog(
'WARNING: Failed to find file in any expected service class - will set STAGE_SVCCLASS to %s'
% castorConfig['svcClassDefault'])
serviceClass = castorConfig['svcClassDefault']
tolog('Setting STAGE_SVCCLASS to %s' % serviceClass)
os.environ['STAGE_SVCCLASS'] = serviceClass
dest_path = os.path.join(path, lfn)
_cmd_str = '%s/usr/bin/rfcp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'local'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-','')
if not path:
tolog('path is empty, using current directory')
path = os.getcwd()
# build setup string
envsetup = self.getEnvsetup(get=True)
# should the root file be copied or read directly by athena?
directIn = False
dInfo = getDirectAccessDic(readpar('copysetupin'))
# if copysetupin did not contain direct access info, try the copysetup instead
if not dInfo:
dInfo = getDirectAccessDic(readpar('copysetup'))
tolog("dInfo: %s" % str(dInfo))
# check if we should use the copytool
if dInfo:
directIn = dInfo['directIn']
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
return 0, self.__pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("not directIn")
# build the get command
_params = ""
if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
csumtype = self.getChecksumType(fchecksum)
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
csumtype = 'md5'
execStr = self.__localget % (envsetup, _params, gpfn, os.path.join(path, lfn))
tolog("Executing command: %s" % (execStr))
report['transferStart'] = time()
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'rfcpLFC'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-','')
tolog("gpfn is %s" % gpfn)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
try:
if not loc_pfn.startswith(('/dpm', '/castor')):
tolog("Potential problem with local filename. Does not start with '/dpm' or '/castor/'.")
except TypeError:
# Older version of python
pass
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
#PN
_cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# if ".lib." in loc_pfn:
# _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# else:
# _cmd_str = '%srfcpXXX %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout
except EnvironmentError, e: # includes OSError (permission) and IOError (write error)
pilotErrorDiag = "put_data mkdirWperm failed: %s" % str(e)
tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
self.prepareReport('MKDIRWPERM_FAIL', report)
return self.put_data_retfail(error.ERR_MKDIR, pilotErrorDiag, surl=dst_gpfn)
report['transferStart'] = time()
# execute dccp
cmd = "%sdccp %s %s" % (_setup_str, source, dst_loc_sedir)
tolog("Executing command: %s" % (cmd))
try:
# t0 = time.time()
# s, cout = commands.getstatusoutput(cmd)
# cerr = ""
# telapsed = time.time() - t0
s, telapsed, cout, cerr = timed_command(cmd, timeout)
print "DEBUG: s = ", s, type(s)
print "DEBUG: telapsed = ", telapsed, type(telapsed)
print "DEBUG: cout = ", cout, type(cout)
print "DEBUG: cerr = ", cerr, type(cerr)
cout = self.filter_text(cout)
cerr = self.filter_text(cerr)
if not self.is_number(s):
s = 1
if not self.is_number(telapsed):
telapsed = 0
except Exception, e:
# There may be exceptions like permission errors
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % (e))
s = 1
e = sys.exc_info()
except ImportError:
import commands
timed_command = None
# Wrapper around commands or timed_command
def run_command(cmd, timeout=0, workdir=None):
"""Run an external command in the workdir directory. Timeout is available only if timed_command is available."""
olddir = None
if workdir:
olddir = os.getcwd()
try:
os.chdir(workdir)
except OSError, e:
return 1, "cd to workdir %s failed: %s" % (workdir, e)
if timed_command:
ec, elapsed, out, err = timed_command.timed_command(cmd, timeout)
outerr = out + err
else:
ec, outerr = commands.getstatusoutput(cmd)
if olddir:
os.chdir(olddir)
return ec, outerr
# Test connection to host:port
def ping(host, port="80"):
"""Check if able to connect to HOST on the specified PORT (80 as default). Return True/False and a message."""
# Simple alive test
# There are more complex implementation of ping running the command in a subprocess,
# as python function, or within a library:
# http://stackoverflow.com/questions/316866/ping-a-site-in-python
# http://www.g-loaded.eu/2009/10/30/python-ping/
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
Fetch a input file
Use uberftp, invoked as shell command, to copy a file on a Gridftp server to a local directory
gpfn -- full URL of the source file (gsiftp://... )
path -- destination directory (local path, absolute or relative to the execution dir, in a local file system).
It is assumed to be there. get_data returns an error if the path is missing
fsize -- source file size, if not provided it is evaluated using uberftp as execution agent
fchecksum -- source file MD5 checsum, if not provided it is evaluated using uberftp as execution agent
"""
error = PilotErrors()
pilotErrorDiag = ""
try:
timeout = pdict['timeout']
except:
timeout = 5*3600
# hot fix for OSCER, remove troubling port number if present
gpfn = gpfn.replace(":2811","")
import socket
if( gpfn.find('SFN') != -1 ):
s = gpfn.split('SFN=')
src_pfn = s[1]
# in this case the file is local
src_host = socket.getfqdn()
else:
# it assumes gpfn to be a URL: the host to have a GSIftp server, and it to see the file in the path of the URL
_tmp = gpfn.split('/', 3)
src_pfn = '/'+_tmp[3]
src_host = _tmp[2]
src_dirname = os.path.dirname(src_pfn)
src_filename = os.path.basename(src_pfn)
dest_file = os.path.join(path, src_filename)
# when the file has been copied we will rename it to the lfn (to remove the __DQ2-string on some files)
final_dest_file = os.path.join(path, lfn)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
# do we have a proxy?
s, pilotErrorDiag = self.verifyProxy(envsetup=_setup_str)
if s != 0:
return s, pilotErrorDiag
# protect against double slashes that might cause problems
# [path] /lscratch/usatlas1_912327//Panda_Pilot... -> /lscratch/usatlas1_912327/Panda_Pilot...
# [src_dirname] //ibrix/data/dq2-cache -> /ibrix/data/dq2-cache
path = path.replace('//', '/')
src_dirname = src_dirname.replace('//', '/')
# download the remote file to the local directory
# if you end the command string with ';' you'll always receive error (ec=1)
_cmd_str_remote = '%suberftp %s "lcd %s; cd %s; get %s; ls %s"|grep %s|grep -v uberftp' %\
(_setup_str, src_host, path, src_dirname, src_filename, src_filename, src_filename)
tolog("Executing command: %s" % (_cmd_str_remote))
try:
s, telapsed, cout, cerr = timed_command(_cmd_str_remote, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
s = 1
o = str(e)
telapsed = timeout
def put_data(self, src_pfn, destination, fsize=0, fchecksum=0, dsname='', extradirs='', **pdict):
"""
Copy all output file to the local SE
Check SiteMover.put_data for a description of the first parameters
src_pfn (source), destination, fsize, fchecksum
dsname -- name of the dataset, used as additional path for the destination file
extradirs -- additional directories
The function tries to create the destination directory if missing
"""
error = PilotErrors()
pilotErrorDiag = ""
try:
timeout = pdict['timeout']
except:
timeout = 5*3600
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
# do we have a proxy?
s, pilotErrorDiag = self.verifyProxy(envsetup=_setup_str, limit=2)
if s != 0:
return self.put_data_retfail(s, pilotErrorDiag)
# hot fix for OSCER, remove troubling port number if present
src_pfn = src_pfn.replace(":2811","")
# preparing variables
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(src_pfn)
if ec != 0:
return self.put_data_retfail(ec, pilotErrorDiag)
dst_se = destination
if dst_se.find('SFN') != -1: # srm://dcsrm.usatlas.bnl.gov:8443/srm/managerv1?SFN=/pnfs/usatlas.bnl.gov/
s = dst_se.split('SFN=')
dst_loc_se = s[1]
dst_prefix = s[0]
else:
_sentries = dst_se.split('/', 3)
dst_serv = _sentries[0] + '//' + _sentries[2] # 'method://host:port' is it always a ftp server? can it be srm? something else?
dst_host = _sentries[2] # host and port
dst_loc_se = '/'+ _sentries[3]
dst_prefix = dst_serv
dst_relative_path = os.path.join(extradirs, dsname)
# dst_loc_se = base directory, not including the destination sub dir
dst_loc_sedir = os.path.join(dst_loc_se, dst_relative_path)
# dst_loc_sedir = full path to the final destination directory
dst_relpath_list = [i for i in dst_relative_path.split('/') if i]
# >>> a='/root/asd/sss'
# >>> b=[i for i in a.split('/') if i]
# ['root', 'asd', 'sss']
src_filename = os.path.basename(src_pfn)
src_dirname = os.path.dirname(src_pfn)
# Destination file name has to be the same, else all the code of the function has to be checked
dst_filename = src_filename
dst_loc_pfn = os.path.join(dst_loc_sedir, dst_filename)
dst_gpfn = dst_prefix + dst_loc_pfn
# get the number of directories that don't have to be created [from the dst_relpath_list]
# TODO: check if the syntax host:port works for uberftp
existing_dir_count_ev,__err = UberftpSiteMover._checkDir(dst_host, dst_loc_se, dst_relpath_list, _setup_str)
if existing_dir_count_ev < 0:
if existing_dir_count_ev == -2:
pilotErrorDiag = ": %s" % (str(__err))
return self.put_data_retfail(error.ERR_PUTTIMEOUT, pilotErrorDiag)
else:
tolog('!!WARNING!!2999!! Control of existing directories failed: %s' % existing_dir_count_ev)
# If you try to copy a file and the directory does not exist the copy fails
# If you try to create an already existing directory you get 'File exists' error
# If the counting of existing directories failed, try to create all of them
existing_dir_count = 0
# to try to copy the file directly (creating no directory)
# existing_dir_count = len(dst_relpath_list)
else:
existing_dir_count = existing_dir_count_ev
# Preparing and executing the transfer command
_start_dir = dst_loc_se
_additional_dir = ''
_relative_dir = ''
# eg. _start_dir = /ibrix/data/dq2-cache
if existing_dir_count < len(dst_relpath_list):
# if existing_dir_count >= 0:
for i in dst_relpath_list:
if existing_dir_count > 0:
# for every loop iteration, add directory i to _start_dir
_start_dir = os.path.join(_start_dir, i)
# eg. _start_dir = /ibrix/data/dq2-cache/testpanda.destDB.72c916bd-c490-461b-8665-9fb9990081cf_sub0
# eg. i = testpanda.destDB.72c916bd-c490-461b-8665-9fb9990081cf_sub0
existing_dir_count -= 1
else:
_relative_dir = os.path.join(_relative_dir, i)
_additional_dir += "mkdir %s;" % (_relative_dir)
_additional_dir += "chmod %s %s;" % (str(oct(PERMISSIONS_DIR)), _relative_dir)
if _relative_dir:
_relative_dir = "cd %s;" % _relative_dir
_ftp_cmd_0 = "cd %s; %s%s" % (_start_dir, _additional_dir, _relative_dir)
else:
# directories are already all there
_ftp_cmd_0 = "cd %s; " % (os.path.join(_start_dir, dst_relative_path))
# protect against double slashes that might cause problems
# [src_dirname] //ibrix/data/dq2-cache -> /ibrix/data/dq2-cache
src_dirname = src_dirname.replace('//', '/')
# uberftp $host "cd $directory; get $fileName; ls $fileName"|grep $fileName|grep -v uberftp > /tmp/fileSize.txt
_ftp_cmd_str = '%s lcd %s; put %s; ls %s' %\
(_ftp_cmd_0, src_dirname, src_filename, dst_filename)
_cmd_str = "%swhich uberftp;uberftp %s '%s'" %\
(_setup_str, dst_host, _ftp_cmd_str)
# local dir should exist
# _cmd_str = Copying uberftp: source /hep/home/gridapp/atlas_app/UberFTP/setup.sh; uberftp tier2-02.ochep.ou.edu
# 'mkdir /ibrix/data/dq2-cache;
# cd /ibrix/data/dq2-cache/testpanda.destDB.72c916bd-c490-461b-8665-9fb9990081cf_sub0;
# lcd /lscratch/usatlas1_207555/Panda_Pilot_12498_1174944165/PandaJob_1280039_1174944165;
# put 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root;
# lls 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root;
# ls 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root;'|
# grep 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root|grep -v uberftp
tolog("Executing command: %s" % (_cmd_str))
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'local', lfn, guid)
if not path:
tolog('path is empty, using current directory')
path = os.getcwd()
# build setup string
envsetup = self.getEnvsetup(get=True)
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return 0, self.__pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("not directIn")
# THE FOLLOWING CODE UNTIL THE END OF THIS FUNCTION SHOULD BE REPLACED WITH NEW CODE FOR GLOBUS-ONLINE
def downloadGC():
"""Download Globus Connect file from Globus Server
"""
arg = ['curl','--connect-timeout','20','--max-time','120','-s','-S','http://confluence.globus.org/download/attachments/14516429/globusconnect','-o','globusconnect']
tolog("Download Arguments: %s" % (arg))
subprocess.call(arg)
#Function to retrieve users proxy from Cern myproxy server.
#The credential should be deposite to the server by the user under the Pilots DN username
#and authoried to retrieve it by the Pilot without passphrase,
#to run Globus Online.
#Code partially extracted from MyproxyUtils.py
#Uses as parameter userID, which is obtained from job info from Panda.
def getproxy(userID):
"""Gets users myproxy from cern server
"""
dn = userID
if dn.count('/CN=') > 1:
first_index = dn.find('/CN=')
second_index = dn.find('/CN=', first_index+1)
dn = dn[0:second_index]
arg = ['myproxy-logon','-s','myproxy.cern.ch','--no_passphrase','-l',dn]
subprocess.call(arg)
#Fuction uses endpoint as parameter, that the user specifies at job submission.
#Corresponds to the name of the endpoint for the working node using Globus-
#Connect. Returns the code for the setup
def createEndpoint(endpoint):
"""Create the endpoint the user specifies and returns the code for setup
"""
arg = ['gsissh', 'cli.globusonline.org', 'endpoint-add','--gc', endpoint]
proc = subprocess.Popen(arg, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
return_code = proc.wait()
i = 0
for line in proc.stdout:
tolog(" %s " % (line.rstrip()))
i += 1
if i == 3:
code = line
return code
#Function uses endpoint as parameter to get the code and run the
#setup in Globus Connect. This connects the endpoint on the working node with
#Globus Online
def setupGC(endpoint):
"""Installing Globus Connect on working node. Creates endpoint, get the setup code and uses it on setup mode
"""
code = createEndpoint(endpoint)
code = code.strip()
arg = ['sh', 'globusconnect', '-setup', code]
tolog("Arguments: %s:" % (arg))
subprocess.call(arg)
#Function that runns Globus Connect on the background of the working node
#to make it a Globus Online Endpoint
def startGC():
"""Start Globus Connect process on working node, on the background
"""
tolog("-----Running Globus Connect------")
arg = ['sh', 'globusconnect', '-start']
subprocess.Popen(arg)
#Function uses source and destinations as parameters.Source is a
#Globus Online Endpoint and path to file, like this:
#SITE1:/home/osg_data/panda/0064.file
#Destination is the Endpoint on the working node ,using Globus Connect,
#like this:
#OSGDEV:/direct/usatlas+u/ccontrer/pilot/autopilot/trunk/
#User defines source and destination at job submission.
#This function activates the source endpoint, under the consideration
#it is a gridFTP server. Destination is already activated since it is a
#Globus Connect endpoint and need no manual activation
#This function creates one transfer task, so it should be executed
#for as many source files as the user specifies
#For a better dinamic, destination path can/should be obtained from job information (?)
#and the endpoint is specified by user, as above.
dest = endpoint + ':' + jobdir
def stageincmdGO(src, dest):
"""create command for stage-in using Globus Online
"""
#Activate the source endpoint
source = src.split(':')[0]
arg = ['gsissh', 'cli.globusonline.org', 'endpoint-activate','-g', source]
subprocess.call(arg)
#Create transfer task
cmd = 'gsissh cli.globusonline.org scp -v %s %s'%(src, dest)
tolog("Transfer task created: %s" % (cmd))
return cmd
#END OF GLOBUS ONLINE FUNCTIONS
# build the get command
_params = ""
if fsize != 0 and fsize != "0":
_params += self.__par_filesize % (fsize,)
if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
csumtype = self.getChecksumType(fchecksum)
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
csumtype = 'md5'
_params += self.__par_checksum % ("%s:%s" % (csumtype, fchecksum),)
# add the guid option
_params += " --guid %s" % (guid)
execStr = self.__localget % (envsetup, _params, gpfn, os.path.join(path, lfn))
tolog("Executing command: %s" % (execStr))
report['transferStart'] = time()
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout
if fchecksum != 0:
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
_csumtype = 'md5'
else:
_csumtype = csumtype
_params += self.__par_checksum % ("%s:%s" % (_csumtype, fchecksum),)
# add the guid option
_params += " --guid %s" % (guid)
execStr = self.__localput % (envsetup, _params, source, dst_gpfn)
tolog("Executing command: %s" % (execStr))
report['transferStart'] = time()
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout
else:
output = cout + cerr
tolog("Elapsed time: %d" % (telapsed))
tolog("Command output:\n%s" % (output))
report['validateStart'] = time()
# validate
if status:
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" copy input file from SE to local dir """
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'local', lfn, guid)
if not path:
tolog('path is empty, using current directory')
path = os.getcwd()
# build setup string
envsetup = self.getEnvsetup(get=True)
ec, pilotErrorDiag = verifySetupCommand(self.__error, envsetup)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return self.__error.ERR_DIRECTIOFILE, self.__pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("not directIn")
# build the get command
_params = ""
if fsize != 0 and fsize != "0":
_params += self.__par_filesize % (fsize,)
if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
csumtype = self.getChecksumType(fchecksum)
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
csumtype = 'md5'
_params += self.__par_checksum % ("%s:%s" % (csumtype, fchecksum),)
# add the guid option
_params += " --guid %s" % (guid)
dest_path = os.path.join(path, lfn)
#PN
# if ".lib." in gpfn:
# localGet = self.__localget
# else:
# localGet = self.__localgetBAD
# execStr = localGet % (envsetup, _params, gpfn, dest_path)
execStr = self.__localget % (envsetup, _params, gpfn, dest_path)
tolog("Executing command: %s" % (execStr))
report['transferStart'] = time()
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5 * 3600)
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time.time()
pnfsid = self.getPnfsid(loc_pfn, guid)
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if pnfsid == None:
isStaged = self.isFileStaged(_setup_str, loc_pfn)
else:
_com = "/cacheinfos/isFileInPool?pnfsid=%s" % (pnfsid)
isStaged = self.isFileStaged(
_setup_str,
loc_pfn,
url="ddmv02.usatlas.bnl.gov:8000",
com=_com)
if not isStaged:
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (
loc_pfn)
self.prepareReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
if pnfsid == None:
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
else:
_cmd_str = '%sdccp pnfs://dcdcap.usatlas.bnl.gov:22125/%s %s' % (
_setup_str, pnfsid, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time.time()
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
str(e))
s = 1
o = str(e)
telapsed = timeout
try:
_dq2SiteName = self.getDQ2SiteName(surl=dst_gpfn)
except Exception, e:
tolog("Warning: Failed to get the DQ2 site name: %s (can not add this info to tracing report)" % str(e))
else:
report['localSite'], report['remoteSite'] = (_dq2SiteName, _dq2SiteName)
tolog("DQ2 site name: %s" % (_dq2SiteName))
# Directory not created: the copy command creates the direcrory if necessary,
# all with the right permission
report['transferStart'] = time()
try:
_cmd_str = '%snecp %s %s' % (_setup_str, source, dst_loc_sedir)
tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
# Command may have exceptions like permission errors
# timed_command hsould block/filter out all exceptions; remove try block?
s = 1
e = sys.exc_info()
o = str(e[1]) #set error message from exception text
telapsed = timeout
else:
o = cout + cerr
tolog("Elapsed time: %d" % (telapsed))
report['validateStart'] = time()
if s != 0:
check_syserr(s, o)
pilotErrorDiag = "Error in copying: %s" % (o)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5*3600)
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time.time()
pnfsid = self.getPnfsid(loc_pfn, guid)
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if pnfsid == None:
isStaged = self.isFileStaged(_setup_str, loc_pfn)
else:
_com = "/cacheinfos/isFileInPool?pnfsid=%s" % (pnfsid)
isStaged = self.isFileStaged(_setup_str, loc_pfn, url="ddmv02.usatlas.bnl.gov:8000", com=_com)
if not isStaged:
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (loc_pfn)
self.__sendReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
if pnfsid == None:
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
else:
_cmd_str = '%sdccp pnfs://dcdcap.usatlas.bnl.gov:22125/%s %s' % (_setup_str, pnfsid, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time.time()
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
Moves a DS file the local SE (where was put from DDM) to the working directory.
Performs the copy and, for systems supporting it, checks size and md5sum correctness
gpfn: full source URL (e.g. method://[host[:port]/full-dir-path/filename - a SRM URL is OK)
path: destination absolute path (in a local file system)
returns the status of the transfer. In case of failure it should remove the partially copied destination
"""
# The local file is assumed to have a relative path that is the same of the relative path in the 'gpfn'
# loc_... are the variables used to access the file in the locally exported file system
error = PilotErrors()
pilotErrorDiag = ""
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
dsname = pdict.get('dsname', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn, guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.__sendReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
tolog("xrootdSiteMover get_data using setup: %s" % (_setup_str))
# remove any host and SFN info from PFN path
src_loc_pfn = self.extractPathFromPFN(gpfn)
src_loc_filename = lfn
# source vars: gpfn, loc_pfn, loc_host, loc_dirname, loc_filename
# dest vars: path
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# protect against bad pfn's
src_loc_pfn = src_loc_pfn.replace('///','/')
src_loc_pfn = src_loc_pfn.replace('//xrootd/','/xrootd/')
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
else:
rootFile = self.isRootFile(src_loc_pfn, setup=_setup_str)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
elif rootFile:
tolog("Found root file: %s (will not be transferred in direct reading mode)" % (src_loc_pfn))
report['relativeStart'] = None
report['transferStart'] = None
self.__sendReport('IS_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("No direct access mode")
ec = 0
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(src_loc_pfn, csumtype=csumtype)
if ec != 0:
self.__sendReport('GET_LOCAL_FILE_INFO_FAIL', report)
return ec, pilotErrorDiag
dest_file = os.path.join(path, src_loc_filename)
report['relativeStart'] = time()
# determine which copy command to use
cpt = self.getCopytool(_setup_str)
report['transferStart'] = time()
cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
#PN
# if ".lib." in src_loc_pfn:
# cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
# else:
# cmd = "%s %sXXX %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
tolog("Executing command: %s" % (cmd))
# execute
timeout = 3600
try:
rc, telapsed, cout, cerr = timed_command(cmd, timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
rc = 1
rs = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
Moves a DS file the local SE (where was put from DDM) to the working directory.
Performs the copy and, for systems supporting it, checks size and md5sum correctness
gpfn: full source URL (e.g. method://[host[:port]/full-dir-path/filename - a SRM URL is OK)
path: destination absolute path (in a local file system)
returns the status of the transfer. In case of failure it should remove the partially copied destination
"""
# The local file is assumed to have a relative path that is the same of the relative path in the 'gpfn'
# loc_... are the variables used to access the file in the locally exported file system
error = PilotErrors()
pilotErrorDiag = ""
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
dsname = pdict.get('dsname', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn,
guid)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = ''
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
tolog("xrootdSiteMover get_data using setup: %s" % (_setup_str))
# remove any host and SFN info from PFN path
src_loc_pfn = self.extractPathFromPFN(gpfn)
src_loc_filename = lfn
# source vars: gpfn, loc_pfn, loc_host, loc_dirname, loc_filename
# dest vars: path
if fchecksum != 0 and fchecksum != "":
csumtype = self.getChecksumType(fchecksum)
else:
csumtype = "default"
# protect against bad pfn's
src_loc_pfn = src_loc_pfn.replace('///', '/')
src_loc_pfn = src_loc_pfn.replace('//xrootd/', '/xrootd/')
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
rootFile = self.isRootFile(src_loc_pfn, setup=_setup_str)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file: %s (will not be transferred in direct reading mode)"
% (src_loc_pfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('IS_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
else:
tolog("No direct access mode")
ec = 0
if fsize == 0 or fchecksum == 0:
ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(
src_loc_pfn, csumtype=csumtype)
if ec != 0:
self.prepareReport('GET_LOCAL_FILE_INFO_FAIL', report)
return ec, pilotErrorDiag
dest_file = os.path.join(path, src_loc_filename)
report['relativeStart'] = time()
# determine which copy command to use
cpt = self.getCopytool(_setup_str)
report['transferStart'] = time()
cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
#PN
# if ".lib." in src_loc_pfn:
# cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
# else:
# cmd = "%s %sXXX %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
tolog("Executing command: %s" % (cmd))
# execute
timeout = 3600
try:
rc, telapsed, cout, cerr = timed_command(cmd, timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(
e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
rc = 1
rs = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5*3600)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'dCache', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time()
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if not self.isFileStaged(_setup_str, loc_pfn):
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (loc_pfn)
self.prepareReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", ftype="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
telapsed = 0
try:
# t0 = time()
# s, cout = commands.getstatusoutput(_cmd_str)
# cerr = ""
# telapsed = time() - t0
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
print "DEBUG: s = ", s, type(s)
print "DEBUG: telapsed = ", telapsed, type(telapsed)
print "DEBUG: cout = ", cout, type(cout)
print "DEBUG: cerr = ", cerr, type(cerr)
cout = self.filter_text(cout)
cerr = self.filter_text(cerr)
if not self.is_number(s):
s = 1
if not self.is_number(telapsed):
telapsed = 0
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % (e))
s = 1
o = self.filter_text(e)
telapsed = timeout
# write traceback info to stderr
import traceback
exc, msg, tb = sys.exc_info()
traceback.print_tb(tb)
_pilotErrorDiag = "Unexpected exception: %s" % (get_exc_plus())
tolog("!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag))
print "!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file (local access to the dCache file) is assumed to have a relative path
that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
analJob = pdict.get('analJob', False)
timeout = pdict.get('timeout', 5 * 3600)
# try to get the direct reading control variable (False for direct reading mode; file should not be copied)
useCT = pdict.get('usect', True)
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'dCache', lfn,
guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
copyprefixin = readpar('copyprefixin')
if copyprefixin != '':
# Extract the copy prefix
pfrom, pto = copyprefixin.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefixin to file: %s" % (loc_pfn))
else:
copyprefix = readpar('copyprefix')
if copyprefix != '':
# Extract the copy prefix
pfrom, pto = copyprefix.split('^')
loc_pfn = pfrom + loc_pfn
tolog("Added copyprefix to file: %s" % (loc_pfn))
report['relativeStart'] = time()
# for analysis jobs, skip input file if on tape or if lib file
if analJob:
if not self.isLibFile(loc_pfn):
if not self.isFileStaged(_setup_str, loc_pfn):
pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (
loc_pfn)
self.prepareReport('FILE_ON_TAPE', report)
return error.ERR_FILEONTAPE, pilotErrorDiag
else:
tolog("Skipping file stage check for lib file")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
ftype="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
ftype="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
_cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
telapsed = 0
try:
# t0 = time()
# s, cout = commands.getstatusoutput(_cmd_str)
# cerr = ""
# telapsed = time() - t0
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
print "DEBUG: s = ", s, type(s)
print "DEBUG: telapsed = ", telapsed, type(telapsed)
print "DEBUG: cout = ", cout, type(cout)
print "DEBUG: cerr = ", cerr, type(cerr)
cout = self.filter_text(cout)
cerr = self.filter_text(cerr)
if not self.is_number(s):
s = 1
if not self.is_number(telapsed):
telapsed = 0
except Exception, e:
tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
(e))
s = 1
o = self.filter_text(e)
telapsed = timeout
# write traceback info to stderr
import traceback
exc, msg, tb = sys.exc_info()
traceback.print_tb(tb)
_pilotErrorDiag = "Unexpected exception: %s" % (get_exc_plus())
tolog("!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag))
print "!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag)
def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
""" copy output file from local dir to SE and register into dataset and catalogues """
# Get input parameters from pdict
lfn = pdict.get('lfn', '')
guid = pdict.get('guid', '')
token = pdict.get('token', '')
dsname = pdict.get('dsname', '')
analJob = pdict.get('analJob', False)
sitename = pdict.get('sitename', '')
extradirs = pdict.get('extradirs', '')
prodSourceLabel = pdict.get('prodSourceLabel', '')
dispatchDBlockTokenForOut = pdict.get('dispatchDBlockTokenForOut', '')
if sitename == "CERNVM" and dispatchDBlockTokenForOut == "":
dispatchDBlockTokenForOut = "chirp^cvmappi50.cern.ch^/panda_test^-d chirp"
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'local'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
report['guid'] = guid.replace('-','')
# report['dataset'] = dsname
filename = os.path.basename(source)
# get the local file size and checksum
csumtype = self.checksum_command
if fsize == 0 or fchecksum == 0:
ec, self.__pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(source, csumtype=csumtype)
if ec != 0:
self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
return self.put_data_retfail(ec, self.__pilotErrorDiag)
# do not transfer files larger than 50 MB except for CERNVM
if sitename != "CERNVM" and int(fsize) > self.__MAX_FILE_SIZE:
self.__pilotErrorDiag = "File sizes larger than %d B can currently not be tranferred with this site mover: size=%s" % (self.__MAX_FILE_SIZE, fsize)
tolog("!!WARNING!!2997!! %s" % (self.__pilotErrorDiag))
return self.put_data_retfail(self.__error.ERR_OUTPUTFILETOOLARGE, self.__pilotErrorDiag)
# now that the file size is known, add it to the tracing report
report['filesize'] = fsize
# build the command
_params = ""
if fchecksum != 0:
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
_csumtype = 'md5'
else:
_csumtype = csumtype
# This contains the user configuration for chirp server, path, debug
# Format should be like
# 'chirp^etpgrid01.garching.physik.uni-muenchen.de^/tanyasandoval^-d chirp'
dispatchDBlockTokenForOut = pdict.get('dispatchDBlockTokenForOut', '')
csplit = dispatchDBlockTokenForOut.split('^')
if len(csplit) != 4:
tolog("Wrong number of fields in chirp string: %s" %
(dispatchDBlockTokenForOut))
self.__pilotErrorDiag = "Wrong number of fields in chirp string: %s" % (dispatchDBlockTokenForOut)
return self.put_data_retfail(self.__error.ERR_STAGEOUTFAILED, self.__pilotErrorDiag)
# Remove _sub part from dataset name
re_sub=re.compile('(.*)_sub\d+')
resub =re_sub.search(dsname)
if resub:
dsname_strip=resub.group(1)
else:
dsname_strip=dsname
chirp_server = csplit[1]
chirp_base = csplit[2]
chirp_options = csplit[3]
chirp_path = chirp_base+'/'+dsname_strip+'/'+filename
tolog("Chirp path: %s" % (chirp_path))
# Make compound command file to run in chirp
chirpcom=open('chirp.com','w')
# Create directories.
dirs=chirp_path.split('/')
dir_path=''
for i in range(1,len(dirs)-1):
dir_path=dir_path+'/'+dirs[i]
chirpcom.write('mkdir '+dir_path+'\n')
# and the cop command too
chirpcom.write('put %s %s\n'%(source,chirp_path))
chirpcom.close()
execStr = self.__chirp % (chirp_options, chirp_server, 'chirp.com')
tolog("Executing command: %s" % (execStr))
try:
status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
except Exception, e:
self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
tolog(self.__warningStr % self.__pilotErrorDiag)
status = 1
output = str(e)
telapsed = self.__timeout
try:
_dq2SiteName = self.getDQ2SiteName(surl=dst_gpfn)
except Exception, e:
tolog("Warning: Failed to get the DQ2 site name: %s (can not add this info to tracing report)" % str(e))
else:
report['localSite'], report['remoteSite'] = (_dq2SiteName, _dq2SiteName)
tolog("DQ2 site name: %s" % (_dq2SiteName))
# Directory not created: the copy command creates the direcrory if necessary,
# all with the right permission
report['transferStart'] = time()
try:
_cmd_str = '%snecp %s %s' % (_setup_str, source, dst_loc_sedir)
tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
# Command may have exceptions like permission errors
# timed_command hsould block/filter out all exceptions; remove try block?
s = 1
e = sys.exc_info()
o = str(e[1]) #set error message from exception text
telapsed = timeout
else:
o = cout + cerr
tolog("Elapsed time: %d" % (telapsed))
report['validateStart'] = time()
if s != 0:
check_syserr(s, o)
pilotErrorDiag = "Error in copying: %s" % (o)
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
""" The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
TODO: document GPFN format (SURL from catalog srm://host/path)
TODO: document better constraint
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the Rucio tracing report
report = self.getStubTracingReport(pdict['report'], 'castorSVC', lfn, guid)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
# Hard code the configuration dictionary for now, but eventually this should be
# set dynamically.
#
# There are the following configuration sections:
# setup - base environment veriables to be set
# svcClassMap - dictionary of string matches vs. service class names
# svcClassList - list of all service classes in case the svcClassMap matching fails
# svcClassDefault - the service class to set if the file appears to be staged no where
#
# Information from RAL:
# [root@srm0661 ~]# listStorageArea -v atlas
# <Space Token> <Description> <service class> <type> <status>
# 4948ef55-0000-1000-b7dd-9b38bdd87201 "ATLASGROUP" "atlasStripDeg" "DURABLE" "ALLOCATED"
# 4948ef38-0000-1000-8606-973e4e998e02 "ATLASMCDISK" "atlasSimStrip" "DURABLE" "ALLOCATED"
# 4948eec6-0000-1000-8ca2-aba0529b4806 "ATLASDATADISK" "atlasStripInput" "DURABLE" "ALLOCATED"
# 4948ee8e-0000-1000-9ac5-81bb9b34ba7b "ATLASMCTAPE" "atlasSimRaw" "PERMANENT" "ALLOCATED"
# 4948ee71-0000-1000-b611-a0afad31f6c8 "ATLASDATATAPE" "atlasT0Raw" "PERMANENT" "ALLOCATED"
# "ATLASHOTDISK" "atlasHotDisk"
# In addition there is the "atlasFarm" class, which is used when data is staged back from tape
castorConfig = {
'setup' : {
'STAGE_HOST' : 'catlasstager.ads.rl.ac.uk',
'STAGER_HOST' : 'catlasstager.ads.rl.ac.uk',
'RFIO_USE_CASTOR_V2' : 'YES',
},
'svcClassList' : ('atlasHotDisk', 'atlasSimStrip', 'atlasStripInput', 'atlasFarm', 'atlasStripDeg', 'atlasT0Raw', 'atlasSimRaw', 'atlasScratchDisk', ),
'svcClassMap' : {
'/atlashotdisk/' : 'atlasHotDisk',
'/atlasmcdisk/' : 'atlasStripInput',
'/atlasdatadisk/' : 'atlasStripInput',
'/atlasgroupdisk/' : 'atlasStripDeg',
'/atlasdatatape/' : 'atlasFarm',
'/atlasmctape/' : 'atlasFarm',
'/atlasscratchdisk/' : 'atlasScratchDisk',
'/atlasProdDisk/' : 'atlasScratchDisk',
},
'svcClassDefault' : 'atlasFarm',
}
# Set all environment variables for castor setup
for envVar, value in castorConfig['setup'].iteritems():
os.environ[envVar] = value
# Strip the gpfn (SURL) back to its bare castor component
tolog("gpfn is %s" % gpfn)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
loc_pfn = ''
if( gpfn.find('SFN') != -1 ):
s = gpfn.split('SFN=')
loc_pfn = s[1]
tolog("Found SFN string. Local file name %s" % loc_pfn)
else:
_tmp = gpfn.split('/', 3)
loc_pfn = '/'+_tmp[3]
tolog("Splitting SURL on slashes. Got local file name %s" % loc_pfn)
if not loc_pfn.startswith('/castor/'):
tolog("WARNING: Problem with local filename: Does not start with '/castor/'.")
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
elif rootFile:
tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", ftype="input")
else:
updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", ftype="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
# Now need to find the service class associated with the file.
# If we find a clear indication of a space token in the file path
# then this is easy. However, if we don't, then use stager_qry to
# interrogate each possible service class. If this fails then use
# atlasFarm in desperation.
serviceClass = None
for pathMatch, svcClass in castorConfig['svcClassMap'].iteritems():
if loc_pfn.find(pathMatch) >= 0:
tolog('Matched path element %s - service class is %s' % (pathMatch, svcClass))
serviceClass = svcClass
break
else:
tolog('Path element %s for service class %s - no match' % (pathMatch, svcClass))
# For testing the fallback, then we need to hobble ourselves by unsetting serviceClass:
#tolog('Automatic service class was: %s' % serviceClass)
#tolog('Unsetting service class for fallback testing')
#serviceClass = None
if serviceClass == None:
tolog("Warning: Failed to find service class hint in SURL.")
for tryMe in castorConfig['svcClassList']:
os.environ['STAGE_SVCCLASS'] = tryMe
tolog('Trying service class %s for file' % tryMe)
err, output = commands.getstatusoutput('stager_qry -M %s' % loc_pfn)
if err != 0:
tolog('WARNING: Unexpected status from stager_qry: %d\n%s' % (err, output))
else:
if output.find('STAGED') >= 0:
tolog('Found file in service class %s' % tryMe)
serviceClass = tryMe
break
else:
tolog('File not found in service class %s' % tryMe)
if serviceClass == None:
tolog('WARNING: Failed to find file in any expected service class - will set STAGE_SVCCLASS to %s' % castorConfig['svcClassDefault'])
serviceClass = castorConfig['svcClassDefault']
tolog('Setting STAGE_SVCCLASS to %s' % serviceClass)
os.environ['STAGE_SVCCLASS'] = serviceClass
dest_path = os.path.join(path, lfn)
_cmd_str = '%s/usr/bin/rfcp %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout
def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
"""
The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
loc_... are the variables used to access the file in the locally exported file system
"""
error = PilotErrors()
pilotErrorDiag = ""
# Get input parameters from pdict
useCT = pdict.get('usect', True)
jobId = pdict.get('jobId', '')
workDir = pdict.get('workDir', '')
prodDBlockToken = pdict.get('access', '')
# get the DQ2 tracing report
try:
report = pdict['report']
except:
report = {}
else:
# set the proper protocol
report['protocol'] = 'rfcpLFC'
# mark the relative start
report['relativeStart'] = time()
# the current file
report['filename'] = lfn
# guid
report['guid'] = guid.replace('-', '')
tolog("gpfn is %s" % gpfn)
# get a proper envsetup
envsetup = self.getEnvsetup(get=True)
if self._setup:
_setup_str = "source %s; " % self._setup
else:
_setup_str = envsetup
ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
if ec != 0:
self.prepareReport('RFCP_FAIL', report)
return ec, pilotErrorDiag
# remove any host and SFN info from PFN path
loc_pfn = self.extractPathFromPFN(gpfn)
try:
if not loc_pfn.startswith(('/dpm', '/castor')):
tolog(
"Potential problem with local filename. Does not start with '/dpm' or '/castor/'."
)
except TypeError:
# Older version of python
pass
# should the root file be copied or read directly by athena?
directIn, useFileStager = self.getTransferModes()
if directIn:
if useCT:
directIn = False
tolog(
"Direct access mode is switched off (file will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
type="input")
else:
# determine if the file is a root file according to its name
rootFile = self.isRootFileName(lfn)
if prodDBlockToken == 'local' or not rootFile:
directIn = False
tolog(
"Direct access mode has been switched off for this file (will be transferred with the copy tool)"
)
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="copy_to_scratch",
type="input")
elif rootFile:
tolog(
"Found root file according to file name: %s (will not be transferred in direct reading mode)"
% (lfn))
report['relativeStart'] = None
report['transferStart'] = None
self.prepareReport('FOUND_ROOT', report)
if useFileStager:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="file_stager",
type="input")
else:
updateFileState(lfn,
workDir,
jobId,
mode="transfer_mode",
state="remote_io",
type="input")
return error.ERR_DIRECTIOFILE, pilotErrorDiag
else:
tolog("Normal file transfer")
dest_path = os.path.join(path, lfn)
#PN
_cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# if ".lib." in loc_pfn:
# _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
# else:
# _cmd_str = '%srfcpXXX %s %s' % (_setup_str, loc_pfn, dest_path)
tolog("Executing command: %s" % (_cmd_str))
report['transferStart'] = time()
# execute
timeout = 3600
try:
s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
except Exception, e:
pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
s = 1
o = str(e)
telapsed = timeout