def stageInFile(self, source, destination, sourceSize, sourceChecksum, guid=None): """StageIn the file. should be implementated by different site mover.""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None # build the parameters _params = "" if sourceSize != 0 and sourceSize != "0": _params += self.__par_filesize % (sourceSize) if sourceChecksum and sourceChecksum != 'None' and sourceChecksum != 0 and sourceChecksum != "0" and not self.isDummyChecksum(sourceChecksum): csumtype = self.getChecksumType(sourceChecksum) # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum') if csumtype == 'md5sum': csumtype = 'md5' _params += self.__par_checksum % ("%s:%s" % (csumtype, sourceChecksum),) # add the guid option _params += " --guid %s" % (guid) self.log("StageIn files started.") _cmd_str = self.__localget % (self._setup, _params, source, destination) self.log('Executing command: %s' % (_cmd_str)) s = -1 o = '(not defined)' t0 = os.times() outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) s, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e))) o = str(e)
def stageOutFile(self, source, destination, sourceSize=None, sourceChecksum=None, token=None, timeout=3600): if self._useTimerCommand: timerCommand = TimerCommand() ret = timerCommand.runFunction(self.s3StageOutFile, args=(source, destination, sourceSize, sourceChecksum, token), timeout=timeout) return ret else: return self.s3StageOutFile(source, destination, sourceSize, sourceChecksum, token)
def removeRemoteFile(self, surl): """ Do remove (remote) file from storage :raise: PilotException in case of controlled error """ # take a 1 m nap before trying to reach the file (it might not be available immediately after a transfer) self.log("INFO: [gfal removeRemoteFile] Taking a 1 m nap before the file removal attempt") time.sleep(60) timeout = self.getTimeOut(0) cmd = 'gfal-rm --verbose -t %s %s' % (timeout, surl) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) self.log("Do remove RemoteFile: %s" % surl) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e)) rcode, output = -1, str(e)
def removeRemoteFile(self, surl): """ Do remove (remote) file from storage :raise: PilotException in case of controlled error """ # take a 1 m nap before trying to reach the file (it might not be available immediately after a transfer) self.log( "INFO: [lcgcp removeRemoteFile] Taking a 1 m nap before the file removal attempt" ) time.sleep(60) timeout = self.getTimeOut(0) cmd = 'lcg-del --vo atlas --verbose -b -l -T srmv2 -t %s --nolfc %s' % ( timeout, surl) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) self.log("Do remove RemoteFile: %s" % surl) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e)) rcode, output = -1, str(e)
def _stagefile(self, source, destination, filesize, is_stagein): """ Stage the file mode is stagein or stageout :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure :raise: PilotException in case of controlled error """ if self.checksum_type not in ["adler32"]: # exclude md5 raise PilotException( "Failed to stage file: internal error: unsupported checksum_type=%s .. " % self.checksum_type, code=PilotErrors.ERR_STAGEINFAILED if is_stagein else PilotErrors.ERR_STAGEOUTFAILED, state="BAD_CSUMTYPE", ) cmd = "%s -np -f %s %s %s" % (self.copy_command, self.coption, source, destination) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) timeout = self.getTimeOut(filesize) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: xrdcp threw an exception: %s" % e) rcode, output = -1, str(e)
def stageInFile(self, source, destination): """StageIn the file. should be implementated by different site mover.""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None self.log("StageIn files started.") _cmd_str = '%s xrdcp -np %s %s' % (self._setup, source, destination) # update job setup script thisExperiment = getExperiment(self.__experiment) # add the full stage-out command to the job setup script to_script = _cmd_str.replace(destination, "`pwd`/%s" % os.path.basename(destination)) to_script = to_script.lstrip(' ') # remove any initial spaces if to_script.startswith('/'): to_script = 'source ' + to_script thisExperiment.updateJobSetupScript(os.path.dirname(destination), to_script=to_script) self.log('Executing command: %s' % (_cmd_str)) s = -1 o = '(not defined)' t0 = os.times() outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) s, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e))) o = str(e)
def _stagefile(self, cmd, source, destination, filesize, is_stagein): """ Stage the file mode is stagein or stageout :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure :raise: PilotException in case of controlled error """ timeout = self.getTimeOut(filesize) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: %s threw an exception: %s" % (self.copy_command, e)) rcode, output = -1, str(e)
def _stagefile(self, cmd, source, destination, filesize, is_stagein): """ Stage the file mode is stagein or stageout :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure :raise: PilotException in case of controlled error """ timeout = self.getTimeOut(filesize) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: %s threw an exception: %s" % (cmd, e)) rcode, output = -1, str(e)
def stageInFile(self, source, destination): """StageIn the file. should be implementated by different site mover.""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None self.log("StageIn files started.") _cmd_str = '%s xrdcp -np %s %s' % (self._setup, source, destination) # update job setup script thisExperiment = getExperiment(self.__experiment) # add the full stage-out command to the job setup script to_script = _cmd_str.replace( destination, "`pwd`/%s" % os.path.basename(destination)) to_script = to_script.lstrip(' ') # remove any initial spaces if to_script.startswith('/'): to_script = 'source ' + to_script thisExperiment.updateJobSetupScript(os.path.dirname(destination), to_script=to_script) self.log('Executing command: %s' % (_cmd_str)) s = -1 o = '(not defined)' t0 = os.times() outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) s, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e))) o = str(e)
def _stagefile(self, source, destination, filesize, is_stagein): """ Stage the file mode is stagein or stageout :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure :raise: PilotException in case of controlled error """ if self.checksum_type not in ['adler32']: # exclude md5 raise PilotException( "Failed to stage file: internal error: unsupported checksum_type=%s .. " % self.checksum_type, code=PilotErrors.ERR_STAGEINFAILED if is_stagein else PilotErrors.ERR_STAGEOUTFAILED, state='BAD_CSUMTYPE') cmd = '%s -np -f %s %s %s' % (self.copy_command, self.coption, source, destination) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) timeout = self.getTimeOut(filesize) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: xrdcp threw an exception: %s" % e) rcode, output = -1, str(e)
def stageOutFile(self, source, destination, sourceSize, sourceChecksum, checksumType, guid, token=None): """Stage out the file. Should be implementated by different site mover""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None # build the parameters _params = "" if token: # Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places) if "dst:" in token: token = token[len('dst:'):] tolog("Dropped dst: part of space token descriptor; token=%s" % (token)) if 'DATADISK' in token: token = "ATLASDATADISK" else: token = "ATLASGROUPDISK" tolog("Space token descriptor reset to: %s" % (token)) _params = self.__spacetoken % (token) if sourceSize != 0 and sourceSize != "0": _params += self.__par_filesize % (sourceSize) if sourceChecksum: _params += self.__par_checksum % ("%s:%s" % (checksumType, sourceChecksum), ) # add the guid option _params += " --guid %s" % (guid) if ".log." in destination: _cmd_str = self.__localput % (self._setup, _params, source, destination) else: _cmd_str = self.__localputBAD % (self._setup, _params, source, destination) tolog("Executing command: %s" % (_cmd_str)) ec = -1 t0 = os.times() o = '(not defined)' outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) ec, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2999!! lsm command threw an exception: %s" % (o)) o = str(e)
def stageOutFile(self, source, destination): """ Stage out the file Should be implementated by different site mover :return: remote file (checksum, checksum_type) in case of success, throw exception in case of failure :raise: PilotException in case of controlled error """ if self.checksum_type not in ['adler32']: # exclude md5 raise PilotException("Failed to stageOutFile(): internal error: unsupported checksum_type=%s .. " % self.checksum_type, code=PilotErrors.ERR_STAGEOUTFAILED, state='BAD_CSUMTYPE') cmd = "%s -h" % self.copy_command setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) self.log("Execute command (%s) to decide which option should be used to calc file checksum.." % cmd) c = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True) output = c.communicate()[0] self.log("status: %s, output: %s" % (c.returncode, output)) coption = "" if c.returncode: self.log('FAILED to execute command=%s: %s' % (cmd, output)) else: if "--cksum" in output: coption = "--cksum %s:print" % self.checksum_type elif "-adler" in output and self.checksum_type == 'adler32': coption = "-adler" elif "-md5" in output and self.checksum_type == 'md5': coption = "-md5" if coption: self.log("Use %s option to get the checksum" % coption) else: self.log("Cannot find neither -adler nor --cksum. will not use checksum") cmd = '%s -np -f %s %s %s' % (self.copy_command, coption, source, destination) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) timeout = self.getTimeOut(os.path.getsize(source)) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: xrdcp threw an exception: %s" % e) rcode, output = -1, str(e)
def stageInFile(self, source, destination, sourceSize=None, sourceChecksum=None): timerCommand = TimerCommand() ret = timerCommand.runFunction(self.s3StageInFile, args=(source, destination, sourceSize, sourceChecksum), timeout=600) return ret
def stageIn(self, source, destination, fspec): """ Query HTTP for etag, then symlink to the pilot working directory. :param source: original file location :param destination: where to create the link :param fspec: dictionary containing destination replicas, scope, lfn :return: destination file details (checksumtype, checksum, size) """ self.log('source: %s' % str(source)) self.log('destination: %s' % str(destination)) self.log('fspec: %s' % str(fspec)) self.log('fspec.scope: %s' % str(fspec.scope)) self.log('fspec.lfn: %s' % str(fspec.lfn)) self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint)) # figure out the HTTP SURL from Rucio from rucio.client import ReplicaClient rc = ReplicaClient() http_surl_reps = [ r for r in rc.list_replicas(dids=[{ 'scope': fspec.scope, 'name': fspec.lfn }], schemes=['https'], rse_expression=fspec.ddmendpoint) ] self.log('http_surl_reps: %s' % http_surl_reps) http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit( '_-')[0] self.log('http_surl: %s' % http_surl) # retrieve the TURL from the webdav etag cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl self.log('ETAG retrieval: %s' % cmd) try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=10) except Exception, e: self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e) raise PilotException('Could not retrieve STORM WebDAV ETag: %s' % e)
def stageOutFile(self, source, destination, sourceSize, sourceChecksum, checksumType, guid, token=None): """Stage out the file. Should be implementated by different site mover""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None # build the parameters _params = "" if token: # Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places) if "dst:" in token: token = token[len('dst:'):] tolog("Dropped dst: part of space token descriptor; token=%s" % (token)) if 'DATADISK' in token: token = "ATLASDATADISK" else: token = "ATLASGROUPDISK" tolog("Space token descriptor reset to: %s" % (token)) _params = self.__spacetoken % (token) if sourceSize != 0 and sourceSize != "0": _params += self.__par_filesize % (sourceSize) if sourceChecksum: _params += self.__par_checksum % ("%s:%s" % (checksumType, sourceChecksum),) # add the guid option _params += " --guid %s" % (guid) if ".log." in destination: _cmd_str = self.__localput % (self._setup, _params, source, destination) else: _cmd_str = self.__localputBAD % (self._setup, _params, source, destination) tolog("Executing command: %s" % (_cmd_str)) ec = -1 t0 = os.times() o = '(not defined)' outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) ec, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2999!! lsm command threw an exception: %s" % (o)) o = str(e)
def stageInFile(self, source, destination, sourceSize, sourceChecksum, guid=None): """StageIn the file. should be implementated by different site mover.""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None # build the parameters _params = "" if sourceSize != 0 and sourceSize != "0": _params += self.__par_filesize % (sourceSize) if sourceChecksum and sourceChecksum != 'None' and sourceChecksum != 0 and sourceChecksum != "0" and not self.isDummyChecksum( sourceChecksum): csumtype = self.getChecksumType(sourceChecksum) # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum') if csumtype == 'md5sum': csumtype = 'md5' _params += self.__par_checksum % ("%s:%s" % (csumtype, sourceChecksum), ) # add the guid option _params += " --guid %s" % (guid) self.log("StageIn files started.") _cmd_str = self.__localget % (self._setup, _params, source, destination) self.log('Executing command: %s' % (_cmd_str)) s = -1 o = '(not defined)' t0 = os.times() outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) s, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e))) o = str(e)
def stageOutFile(self, source, destination, token=None): """Stage out the file. Should be implementated by different site mover""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None outputRet["output"] = None command = "%s xrdcp -h" % (self._setup) tolog( "Execute command(%s) to decide whether -adler or --cksum adler32 to be used." % command) status_local, output_local = commands.getstatusoutput(command) tolog("status: %s, output: %s" % (status_local, output_local)) checksum_option = "" if "-adler" in output_local: checksum_option = " -adler " elif "--cksum" in output_local: checksum_option = " --cksum adler32 " #checksum_option = " -adler " # currently use this one. --cksum will fail on some sites if checksum_option != "": tolog("Use (%s) to get the checksum" % checksum_option) else: tolog("Cannot find -adler nor --cksum. will not use checksum") #checksum_option = " -adler " # currently use this one. --cksum will fail on some sites # surl is the same as putfile _cmd_str = '%s xrdcp -np -f %s %s %s' % (self._setup, checksum_option, source, destination) tolog("Executing command: %s" % (_cmd_str)) ec = -1 t0 = os.times() o = '(not defined)' outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) ec, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2999!! xrdcp threw an exception: %s" % (o)) o = str(e)
def stageOutFile(self, source, destination, token=None): """Stage out the file. Should be implementated by different site mover""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None outputRet["output"] = None command = "%s xrdcp -h" % (self._setup) tolog("Execute command(%s) to decide whether -adler or --cksum adler32 to be used." % command) status_local, output_local = commands.getstatusoutput(command) tolog("status: %s, output: %s" % (status_local, output_local)) checksum_option = "" if "-adler" in output_local: checksum_option = " -adler " elif "--cksum" in output_local: checksum_option = " --cksum adler32 " #checksum_option = " -adler " # currently use this one. --cksum will fail on some sites if checksum_option != "": tolog("Use (%s) to get the checksum" % checksum_option) else: tolog("Cannot find -adler nor --cksum. will not use checksum") #checksum_option = " -adler " # currently use this one. --cksum will fail on some sites # surl is the same as putfile _cmd_str = '%s xrdcp -np -f %s %s %s' % (self._setup, checksum_option, source, destination) tolog("Executing command: %s" % (_cmd_str)) ec = -1 t0 = os.times() o = '(not defined)' outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) ec, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2999!! xrdcp threw an exception: %s" % (o)) o = str(e)
def isFileStaged(self, fspec): is_staged = True # assume file is staged by default cmd = '%s -P -t -1 %s' % (self.copy_command, fspec.turl) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) timeout = 10 self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: %s threw an exception: %s" % (self.copy_command, e)) rcode, output = -1, str(e)
def stageIn(self, source, destination, fspec): """ Query HTTP for etag, then symlink to the pilot working directory. :param source: original file location :param destination: where to create the link :param fspec: dictionary containing destination replicas, scope, lfn :return: destination file details (checksumtype, checksum, size) """ self.log('source: %s' % str(source)) self.log('destination: %s' % str(destination)) self.log('fspec: %s' % str(fspec)) self.log('fspec.scope: %s' % str(fspec.scope)) self.log('fspec.lfn: %s' % str(fspec.lfn)) self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint)) # figure out the HTTP SURL from Rucio from rucio.client import ReplicaClient rc = ReplicaClient() http_surl_reps = [r for r in rc.list_replicas(dids=[{'scope': fspec.scope, 'name': fspec.lfn}], schemes=['davs'], rse_expression=fspec.ddmendpoint)] self.log('http_surl_reps: %s' % http_surl_reps) http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit('_-')[0] self.log('http_surl: %s' % http_surl) # retrieve the TURL from the webdav etag cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl self.log('ETAG retrieval: %s' % cmd) try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=10) except Exception, e: self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e) raise PilotException('Could not retrieve STORM WebDAV ETag: %s' % e)
def stageInFile(self, source, destination): """StageIn the file. should be implementated by different site mover.""" statusRet = 0 outputRet = {} outputRet["errorLog"] = None outputRet["report"] = {} outputRet["report"]["clientState"] = None self.log("StageIn files started.") _cmd_str = '%s xrdcp %s %s' % (self._setup, source, destination) self.log('Executing command: %s, timeout: %s' % (_cmd_str, self.timeout)) s = -1 o = '(not defined)' t0 = os.times() outputRet["report"]['relativeStart'] = time() outputRet["report"]['transferStart'] = time() try: timerCommand = TimerCommand(_cmd_str) s, o = timerCommand.run(timeout=self.timeout) except Exception, e: tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e))) o = str(e)
def getRemoteFileSize(self, filename): """ get size of remote file Should be implemented by different site mover :return: length of file :raise: an exception in case of errors """ # do not rely on gfal-ls internal time out timeout = self.getTimeOut(0) cmd = "gfal-ls -l -t %d %s" % (timeout, filename) self.log("getRemoteFileSize: execute command: %s" % cmd ) timer = TimerCommand(cmd) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e)) rcode, output = -1, str(e)
def getRemoteFileSize(self, filename): """ get size of remote file Should be implemented by different site mover :return: length of file :raise: an exception in case of errors """ # do not rely on gfal-ls internal time out timeout = self.getTimeOut(0) cmd = "gfal-ls -l -t %d %s" % (timeout, filename) self.log("getRemoteFileSize: execute command: %s" % cmd) timer = TimerCommand(cmd) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e)) rcode, output = -1, str(e)
def handleGfalFile(self, gfalFile): try: for i in range(3): gfalFile = os.path.join(self.__workDir, gfalFile) os.rename(gfalFile, gfalFile + "copying") handle = open(gfalFile + "copying") cmd = handle.read() handle.close() cmd = cmd.replace(" -t 3600 ", " -t 300 ") logging.info("Execute command: %s" % cmd) # status, output = commands.getstatusoutput(cmd) status, output = TimerCommand(cmd).run(600) logging.info("Status %s output %s" % (status, output)) if status == 0: os.rename(gfalFile + "copying", gfalFile + "finished") return else: os.rename(gfalFile + "copying", gfalFile) os.rename(gfalFile, gfalFile + "failed") except: logging.error("handleGfalFile %s" % traceback.format_exc()) finally: self.__handlingOthers -= 1
def stageOutFile(self, source, destination): """ Stage out the file Should be implementated by different site mover :return: remote file (checksum, checksum_type) in case of success, throw exception in case of failure :raise: PilotException in case of controlled error """ if self.checksum_type not in ["adler32"]: # exclude md5 raise PilotException( "Failed to stageOutFile(): internal error: unsupported checksum_type=%s .. " % self.checksum_type, code=PilotErrors.ERR_STAGEOUTFAILED, state="BAD_CSUMTYPE", ) cmd = "%s -h" % self.copy_command setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) self.log("Execute command (%s) to decide which option should be used to calc file checksum.." % cmd) c = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True) output = c.communicate()[0] self.log("status: %s, output: %s" % (c.returncode, output)) coption = "" if c.returncode: self.log("FAILED to execute command=%s: %s" % (cmd, output)) else: if "--cksum" in output: coption = "--cksum %s:print" % self.checksum_type elif "-adler" in output and self.checksum_type == "adler32": coption = "-adler" elif "-md5" in output and self.checksum_type == "md5": coption = "-md5" if coption: self.log("Use %s option to get the checksum" % coption) else: self.log("Cannot find neither -adler nor --cksum. will not use checksum") cmd = "%s -np -f %s %s %s" % (self.copy_command, coption, source, destination) setup = self.getSetup() if setup: cmd = "%s; %s" % (setup, cmd) timeout = self.getTimeOut(os.path.getsize(source)) self.log("Executing command: %s, timeout=%s" % (cmd, timeout)) t0 = datetime.now() is_timeout = False try: timer = TimerCommand(cmd) rcode, output = timer.run(timeout=timeout) is_timeout = timer.is_timeout except Exception, e: self.log("WARNING: xrdcp threw an exception: %s" % e) rcode, output = -1, str(e)