def stageInFile(self, source, destination, sourceSize, sourceChecksum, guid=None):
"""StageIn the file. should be implementated by different site mover."""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
# build the parameters
_params = ""
if sourceSize != 0 and sourceSize != "0":
_params += self.__par_filesize % (sourceSize)
if sourceChecksum and sourceChecksum != 'None' and sourceChecksum != 0 and sourceChecksum != "0" and not self.isDummyChecksum(sourceChecksum):
csumtype = self.getChecksumType(sourceChecksum)
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
csumtype = 'md5'
_params += self.__par_checksum % ("%s:%s" % (csumtype, sourceChecksum),)
# add the guid option
_params += " --guid %s" % (guid)
self.log("StageIn files started.")
_cmd_str = self.__localget % (self._setup, _params, source, destination)
self.log('Executing command: %s' % (_cmd_str))
s = -1
o = '(not defined)'
t0 = os.times()
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
s, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e)))
o = str(e)
def stageOutFile(self, source, destination, sourceSize=None, sourceChecksum=None, token=None, timeout=3600):
if self._useTimerCommand:
timerCommand = TimerCommand()
ret = timerCommand.runFunction(self.s3StageOutFile, args=(source, destination, sourceSize, sourceChecksum, token), timeout=timeout)
return ret
else:
return self.s3StageOutFile(source, destination, sourceSize, sourceChecksum, token)
def removeRemoteFile(self, surl):
"""
Do remove (remote) file from storage
:raise: PilotException in case of controlled error
"""
# take a 1 m nap before trying to reach the file (it might not be available immediately after a transfer)
self.log("INFO: [gfal removeRemoteFile] Taking a 1 m nap before the file removal attempt")
time.sleep(60)
timeout = self.getTimeOut(0)
cmd = 'gfal-rm --verbose -t %s %s' % (timeout, surl)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
self.log("Do remove RemoteFile: %s" % surl)
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e))
rcode, output = -1, str(e)
def removeRemoteFile(self, surl):
"""
Do remove (remote) file from storage
:raise: PilotException in case of controlled error
"""
# take a 1 m nap before trying to reach the file (it might not be available immediately after a transfer)
self.log(
"INFO: [lcgcp removeRemoteFile] Taking a 1 m nap before the file removal attempt"
)
time.sleep(60)
timeout = self.getTimeOut(0)
cmd = 'lcg-del --vo atlas --verbose -b -l -T srmv2 -t %s --nolfc %s' % (
timeout, surl)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
self.log("Do remove RemoteFile: %s" % surl)
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e))
rcode, output = -1, str(e)
def _stagefile(self, source, destination, filesize, is_stagein):
"""
Stage the file
mode is stagein or stageout
:return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
:raise: PilotException in case of controlled error
"""
if self.checksum_type not in ["adler32"]: # exclude md5
raise PilotException(
"Failed to stage file: internal error: unsupported checksum_type=%s .. " % self.checksum_type,
code=PilotErrors.ERR_STAGEINFAILED if is_stagein else PilotErrors.ERR_STAGEOUTFAILED,
state="BAD_CSUMTYPE",
)
cmd = "%s -np -f %s %s %s" % (self.copy_command, self.coption, source, destination)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
timeout = self.getTimeOut(filesize)
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: xrdcp threw an exception: %s" % e)
rcode, output = -1, str(e)
def stageInFile(self, source, destination):
"""StageIn the file. should be implementated by different site mover."""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
self.log("StageIn files started.")
_cmd_str = '%s xrdcp -np %s %s' % (self._setup, source, destination)
# update job setup script
thisExperiment = getExperiment(self.__experiment)
# add the full stage-out command to the job setup script
to_script = _cmd_str.replace(destination, "`pwd`/%s" % os.path.basename(destination))
to_script = to_script.lstrip(' ') # remove any initial spaces
if to_script.startswith('/'):
to_script = 'source ' + to_script
thisExperiment.updateJobSetupScript(os.path.dirname(destination), to_script=to_script)
self.log('Executing command: %s' % (_cmd_str))
s = -1
o = '(not defined)'
t0 = os.times()
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
s, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e)))
o = str(e)
def _stagefile(self, cmd, source, destination, filesize, is_stagein):
"""
Stage the file
mode is stagein or stageout
:return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
:raise: PilotException in case of controlled error
"""
timeout = self.getTimeOut(filesize)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: %s threw an exception: %s" %
(self.copy_command, e))
rcode, output = -1, str(e)
def _stagefile(self, cmd, source, destination, filesize, is_stagein):
"""
Stage the file
mode is stagein or stageout
:return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
:raise: PilotException in case of controlled error
"""
timeout = self.getTimeOut(filesize)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: %s threw an exception: %s" % (cmd, e))
rcode, output = -1, str(e)
def stageInFile(self, source, destination):
"""StageIn the file. should be implementated by different site mover."""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
self.log("StageIn files started.")
_cmd_str = '%s xrdcp -np %s %s' % (self._setup, source, destination)
# update job setup script
thisExperiment = getExperiment(self.__experiment)
# add the full stage-out command to the job setup script
to_script = _cmd_str.replace(
destination, "`pwd`/%s" % os.path.basename(destination))
to_script = to_script.lstrip(' ') # remove any initial spaces
if to_script.startswith('/'):
to_script = 'source ' + to_script
thisExperiment.updateJobSetupScript(os.path.dirname(destination),
to_script=to_script)
self.log('Executing command: %s' % (_cmd_str))
s = -1
o = '(not defined)'
t0 = os.times()
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
s, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" %
(str(e)))
o = str(e)
def _stagefile(self, source, destination, filesize, is_stagein):
"""
Stage the file
mode is stagein or stageout
:return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
:raise: PilotException in case of controlled error
"""
if self.checksum_type not in ['adler32']: # exclude md5
raise PilotException(
"Failed to stage file: internal error: unsupported checksum_type=%s .. "
% self.checksum_type,
code=PilotErrors.ERR_STAGEINFAILED
if is_stagein else PilotErrors.ERR_STAGEOUTFAILED,
state='BAD_CSUMTYPE')
cmd = '%s -np -f %s %s %s' % (self.copy_command, self.coption, source,
destination)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
timeout = self.getTimeOut(filesize)
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: xrdcp threw an exception: %s" % e)
rcode, output = -1, str(e)
def stageOutFile(self,
source,
destination,
sourceSize,
sourceChecksum,
checksumType,
guid,
token=None):
"""Stage out the file. Should be implementated by different site mover"""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
# build the parameters
_params = ""
if token:
# Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places)
if "dst:" in token:
token = token[len('dst:'):]
tolog("Dropped dst: part of space token descriptor; token=%s" %
(token))
if 'DATADISK' in token:
token = "ATLASDATADISK"
else:
token = "ATLASGROUPDISK"
tolog("Space token descriptor reset to: %s" % (token))
_params = self.__spacetoken % (token)
if sourceSize != 0 and sourceSize != "0":
_params += self.__par_filesize % (sourceSize)
if sourceChecksum:
_params += self.__par_checksum % ("%s:%s" %
(checksumType, sourceChecksum), )
# add the guid option
_params += " --guid %s" % (guid)
if ".log." in destination:
_cmd_str = self.__localput % (self._setup, _params, source,
destination)
else:
_cmd_str = self.__localputBAD % (self._setup, _params, source,
destination)
tolog("Executing command: %s" % (_cmd_str))
ec = -1
t0 = os.times()
o = '(not defined)'
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
ec, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2999!! lsm command threw an exception: %s" % (o))
o = str(e)
def stageOutFile(self, source, destination):
"""
Stage out the file
Should be implementated by different site mover
:return: remote file (checksum, checksum_type) in case of success, throw exception in case of failure
:raise: PilotException in case of controlled error
"""
if self.checksum_type not in ['adler32']: # exclude md5
raise PilotException("Failed to stageOutFile(): internal error: unsupported checksum_type=%s .. " % self.checksum_type, code=PilotErrors.ERR_STAGEOUTFAILED, state='BAD_CSUMTYPE')
cmd = "%s -h" % self.copy_command
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
self.log("Execute command (%s) to decide which option should be used to calc file checksum.." % cmd)
c = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
output = c.communicate()[0]
self.log("status: %s, output: %s" % (c.returncode, output))
coption = ""
if c.returncode:
self.log('FAILED to execute command=%s: %s' % (cmd, output))
else:
if "--cksum" in output:
coption = "--cksum %s:print" % self.checksum_type
elif "-adler" in output and self.checksum_type == 'adler32':
coption = "-adler"
elif "-md5" in output and self.checksum_type == 'md5':
coption = "-md5"
if coption:
self.log("Use %s option to get the checksum" % coption)
else:
self.log("Cannot find neither -adler nor --cksum. will not use checksum")
cmd = '%s -np -f %s %s %s' % (self.copy_command, coption, source, destination)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
timeout = self.getTimeOut(os.path.getsize(source))
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: xrdcp threw an exception: %s" % e)
rcode, output = -1, str(e)
def stageInFile(self,
source,
destination,
sourceSize=None,
sourceChecksum=None):
timerCommand = TimerCommand()
ret = timerCommand.runFunction(self.s3StageInFile,
args=(source, destination, sourceSize,
sourceChecksum),
timeout=600)
return ret
def stageIn(self, source, destination, fspec):
"""
Query HTTP for etag, then symlink to the pilot working directory.
:param source: original file location
:param destination: where to create the link
:param fspec: dictionary containing destination replicas, scope, lfn
:return: destination file details (checksumtype, checksum, size)
"""
self.log('source: %s' % str(source))
self.log('destination: %s' % str(destination))
self.log('fspec: %s' % str(fspec))
self.log('fspec.scope: %s' % str(fspec.scope))
self.log('fspec.lfn: %s' % str(fspec.lfn))
self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint))
# figure out the HTTP SURL from Rucio
from rucio.client import ReplicaClient
rc = ReplicaClient()
http_surl_reps = [
r for r in rc.list_replicas(dids=[{
'scope': fspec.scope,
'name': fspec.lfn
}],
schemes=['https'],
rse_expression=fspec.ddmendpoint)
]
self.log('http_surl_reps: %s' % http_surl_reps)
http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit(
'_-')[0]
self.log('http_surl: %s' % http_surl)
# retrieve the TURL from the webdav etag
cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl
self.log('ETAG retrieval: %s' % cmd)
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=10)
except Exception, e:
self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e)
raise PilotException('Could not retrieve STORM WebDAV ETag: %s' %
e)
def stageOutFile(self, source, destination, sourceSize, sourceChecksum, checksumType, guid, token=None):
"""Stage out the file. Should be implementated by different site mover"""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
# build the parameters
_params = ""
if token:
# Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places)
if "dst:" in token:
token = token[len('dst:'):]
tolog("Dropped dst: part of space token descriptor; token=%s" % (token))
if 'DATADISK' in token:
token = "ATLASDATADISK"
else:
token = "ATLASGROUPDISK"
tolog("Space token descriptor reset to: %s" % (token))
_params = self.__spacetoken % (token)
if sourceSize != 0 and sourceSize != "0":
_params += self.__par_filesize % (sourceSize)
if sourceChecksum:
_params += self.__par_checksum % ("%s:%s" % (checksumType, sourceChecksum),)
# add the guid option
_params += " --guid %s" % (guid)
if ".log." in destination:
_cmd_str = self.__localput % (self._setup, _params, source, destination)
else:
_cmd_str = self.__localputBAD % (self._setup, _params, source, destination)
tolog("Executing command: %s" % (_cmd_str))
ec = -1
t0 = os.times()
o = '(not defined)'
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
ec, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2999!! lsm command threw an exception: %s" % (o))
o = str(e)
def stageInFile(self,
source,
destination,
sourceSize,
sourceChecksum,
guid=None):
"""StageIn the file. should be implementated by different site mover."""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
# build the parameters
_params = ""
if sourceSize != 0 and sourceSize != "0":
_params += self.__par_filesize % (sourceSize)
if sourceChecksum and sourceChecksum != 'None' and sourceChecksum != 0 and sourceChecksum != "0" and not self.isDummyChecksum(
sourceChecksum):
csumtype = self.getChecksumType(sourceChecksum)
# special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
if csumtype == 'md5sum':
csumtype = 'md5'
_params += self.__par_checksum % ("%s:%s" %
(csumtype, sourceChecksum), )
# add the guid option
_params += " --guid %s" % (guid)
self.log("StageIn files started.")
_cmd_str = self.__localget % (self._setup, _params, source,
destination)
self.log('Executing command: %s' % (_cmd_str))
s = -1
o = '(not defined)'
t0 = os.times()
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
s, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" %
(str(e)))
o = str(e)
def stageOutFile(self, source, destination, token=None):
"""Stage out the file. Should be implementated by different site mover"""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
outputRet["output"] = None
command = "%s xrdcp -h" % (self._setup)
tolog(
"Execute command(%s) to decide whether -adler or --cksum adler32 to be used."
% command)
status_local, output_local = commands.getstatusoutput(command)
tolog("status: %s, output: %s" % (status_local, output_local))
checksum_option = ""
if "-adler" in output_local:
checksum_option = " -adler "
elif "--cksum" in output_local:
checksum_option = " --cksum adler32 "
#checksum_option = " -adler " # currently use this one. --cksum will fail on some sites
if checksum_option != "":
tolog("Use (%s) to get the checksum" % checksum_option)
else:
tolog("Cannot find -adler nor --cksum. will not use checksum")
#checksum_option = " -adler " # currently use this one. --cksum will fail on some sites
# surl is the same as putfile
_cmd_str = '%s xrdcp -np -f %s %s %s' % (self._setup, checksum_option,
source, destination)
tolog("Executing command: %s" % (_cmd_str))
ec = -1
t0 = os.times()
o = '(not defined)'
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
ec, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2999!! xrdcp threw an exception: %s" % (o))
o = str(e)
def stageOutFile(self, source, destination, token=None):
"""Stage out the file. Should be implementated by different site mover"""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
outputRet["output"] = None
command = "%s xrdcp -h" % (self._setup)
tolog("Execute command(%s) to decide whether -adler or --cksum adler32 to be used." % command)
status_local, output_local = commands.getstatusoutput(command)
tolog("status: %s, output: %s" % (status_local, output_local))
checksum_option = ""
if "-adler" in output_local:
checksum_option = " -adler "
elif "--cksum" in output_local:
checksum_option = " --cksum adler32 "
#checksum_option = " -adler " # currently use this one. --cksum will fail on some sites
if checksum_option != "":
tolog("Use (%s) to get the checksum" % checksum_option)
else:
tolog("Cannot find -adler nor --cksum. will not use checksum")
#checksum_option = " -adler " # currently use this one. --cksum will fail on some sites
# surl is the same as putfile
_cmd_str = '%s xrdcp -np -f %s %s %s' % (self._setup, checksum_option, source, destination)
tolog("Executing command: %s" % (_cmd_str))
ec = -1
t0 = os.times()
o = '(not defined)'
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
ec, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2999!! xrdcp threw an exception: %s" % (o))
o = str(e)
def isFileStaged(self, fspec):
is_staged = True # assume file is staged by default
cmd = '%s -P -t -1 %s' % (self.copy_command, fspec.turl)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
timeout = 10
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: %s threw an exception: %s" % (self.copy_command, e))
rcode, output = -1, str(e)
def stageIn(self, source, destination, fspec):
"""
Query HTTP for etag, then symlink to the pilot working directory.
:param source: original file location
:param destination: where to create the link
:param fspec: dictionary containing destination replicas, scope, lfn
:return: destination file details (checksumtype, checksum, size)
"""
self.log('source: %s' % str(source))
self.log('destination: %s' % str(destination))
self.log('fspec: %s' % str(fspec))
self.log('fspec.scope: %s' % str(fspec.scope))
self.log('fspec.lfn: %s' % str(fspec.lfn))
self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint))
# figure out the HTTP SURL from Rucio
from rucio.client import ReplicaClient
rc = ReplicaClient()
http_surl_reps = [r for r in rc.list_replicas(dids=[{'scope': fspec.scope,
'name': fspec.lfn}],
schemes=['davs'],
rse_expression=fspec.ddmendpoint)]
self.log('http_surl_reps: %s' % http_surl_reps)
http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit('_-')[0]
self.log('http_surl: %s' % http_surl)
# retrieve the TURL from the webdav etag
cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl
self.log('ETAG retrieval: %s' % cmd)
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=10)
except Exception, e:
self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e)
raise PilotException('Could not retrieve STORM WebDAV ETag: %s' % e)
def stageInFile(self, source, destination):
"""StageIn the file. should be implementated by different site mover."""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
self.log("StageIn files started.")
_cmd_str = '%s xrdcp %s %s' % (self._setup, source, destination)
self.log('Executing command: %s, timeout: %s' % (_cmd_str, self.timeout))
s = -1
o = '(not defined)'
t0 = os.times()
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
s, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e)))
o = str(e)
def getRemoteFileSize(self, filename):
"""
get size of remote file
Should be implemented by different site mover
:return: length of file
:raise: an exception in case of errors
"""
# do not rely on gfal-ls internal time out
timeout = self.getTimeOut(0)
cmd = "gfal-ls -l -t %d %s" % (timeout, filename)
self.log("getRemoteFileSize: execute command: %s" % cmd )
timer = TimerCommand(cmd)
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e))
rcode, output = -1, str(e)
def getRemoteFileSize(self, filename):
"""
get size of remote file
Should be implemented by different site mover
:return: length of file
:raise: an exception in case of errors
"""
# do not rely on gfal-ls internal time out
timeout = self.getTimeOut(0)
cmd = "gfal-ls -l -t %d %s" % (timeout, filename)
self.log("getRemoteFileSize: execute command: %s" % cmd)
timer = TimerCommand(cmd)
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e))
rcode, output = -1, str(e)
def stageInFile(self, source, destination):
"""StageIn the file. should be implementated by different site mover."""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
self.log("StageIn files started.")
_cmd_str = '%s xrdcp %s %s' % (self._setup, source, destination)
self.log('Executing command: %s, timeout: %s' %
(_cmd_str, self.timeout))
s = -1
o = '(not defined)'
t0 = os.times()
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
timerCommand = TimerCommand(_cmd_str)
s, o = timerCommand.run(timeout=self.timeout)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" %
(str(e)))
o = str(e)
def handleGfalFile(self, gfalFile):
try:
for i in range(3):
gfalFile = os.path.join(self.__workDir, gfalFile)
os.rename(gfalFile, gfalFile + "copying")
handle = open(gfalFile + "copying")
cmd = handle.read()
handle.close()
cmd = cmd.replace(" -t 3600 ", " -t 300 ")
logging.info("Execute command: %s" % cmd)
# status, output = commands.getstatusoutput(cmd)
status, output = TimerCommand(cmd).run(600)
logging.info("Status %s output %s" % (status, output))
if status == 0:
os.rename(gfalFile + "copying", gfalFile + "finished")
return
else:
os.rename(gfalFile + "copying", gfalFile)
os.rename(gfalFile, gfalFile + "failed")
except:
logging.error("handleGfalFile %s" % traceback.format_exc())
finally:
self.__handlingOthers -= 1
def stageOutFile(self, source, destination):
"""
Stage out the file
Should be implementated by different site mover
:return: remote file (checksum, checksum_type) in case of success, throw exception in case of failure
:raise: PilotException in case of controlled error
"""
if self.checksum_type not in ["adler32"]: # exclude md5
raise PilotException(
"Failed to stageOutFile(): internal error: unsupported checksum_type=%s .. " % self.checksum_type,
code=PilotErrors.ERR_STAGEOUTFAILED,
state="BAD_CSUMTYPE",
)
cmd = "%s -h" % self.copy_command
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
self.log("Execute command (%s) to decide which option should be used to calc file checksum.." % cmd)
c = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
output = c.communicate()[0]
self.log("status: %s, output: %s" % (c.returncode, output))
coption = ""
if c.returncode:
self.log("FAILED to execute command=%s: %s" % (cmd, output))
else:
if "--cksum" in output:
coption = "--cksum %s:print" % self.checksum_type
elif "-adler" in output and self.checksum_type == "adler32":
coption = "-adler"
elif "-md5" in output and self.checksum_type == "md5":
coption = "-md5"
if coption:
self.log("Use %s option to get the checksum" % coption)
else:
self.log("Cannot find neither -adler nor --cksum. will not use checksum")
cmd = "%s -np -f %s %s %s" % (self.copy_command, coption, source, destination)
setup = self.getSetup()
if setup:
cmd = "%s; %s" % (setup, cmd)
timeout = self.getTimeOut(os.path.getsize(source))
self.log("Executing command: %s, timeout=%s" % (cmd, timeout))
t0 = datetime.now()
is_timeout = False
try:
timer = TimerCommand(cmd)
rcode, output = timer.run(timeout=timeout)
is_timeout = timer.is_timeout
except Exception, e:
self.log("WARNING: xrdcp threw an exception: %s" % e)
rcode, output = -1, str(e)
def stageInFile(self, source, destination, sourceSize=None, sourceChecksum=None):
timerCommand = TimerCommand()
ret = timerCommand.runFunction(self.s3StageInFile, args=(source, destination, sourceSize, sourceChecksum), timeout=600)
return ret
