def execute(self, conf=None, caller=None, apMon=None, logger=None):
"""This method is invoked by fdtcp (locally)."""
# fileNameToStoreRemoteUserName - file into which AuthClient
# stores name of the Grid user at the remote party, this
# information is then forwarded later to
# fdtd which doens't have to do the user mapping lookup again
fileName = "/tmp/" + self.id + "--" + getRandomString('a', 'z', 5)
if os.path.exists(fileName):
raise FDTCopyException("File %s exists." % fileName)
self._setUp(conf, fileName)
executor = Executor(self.id,
self.command,
blocking=True,
logger=logger)
# here the Java AuthClient stores the Grid user name into the file
output = executor.execute()
try:
remoteGridUser = open(fileName, 'r').read()
os.remove(fileName)
except Exception, ex:
m = ("Problem handling file %s (reading remote Grid user "
"name), reason: %s" % (fileName, ex))
raise FDTCopyException(m)
def perform(self, action):
result = self.call(action)
if result.status == 0:
self.logger.info("Success, response: %s" % result)
return result
else:
m = ("Error occurred on host %s id: %s, reason: %s\n"
"remote log:\n%s\n" %
(result.host, result.id, result.msg, result.log))
raise FDTCopyException(m)
def __init__(self, uri, logger):
self.logger = logger
self.uri = uri
self.logger.debug("%s creating PYRO proxy for URI: '%s'" %
(self.__class__.__name__, self.uri))
try:
Pyro.core.initClient()
self.proxy = Pyro.core.getProxyForURI(self.uri)
except (AttributeError, URIError), ex:
m = ("Incorrect URI '%s', could not create PYRO proxy for "
"FDTD service." % self.uri)
raise FDTCopyException(m)
def _processCopyJobFile(self, conf, apMon, logger):
"""
Process copyjobfile - list of pairs urlSrc urlDest
(resp. FROM_PFN TO_PFN).
"""
fileName = conf.get("copyjobfile")
try:
lines = open(fileName, 'r').readlines()
except IOError, ex:
raise FDTCopyException("Can't read '%s', reason: %s" %
(fileName, ex))
def _setTransfer(self, urlSrc, urlDest, apMon, conf, logger):
"""
Initialise instance of Transfer class and set up properties.
URL form:
fdt://gridftp01.ultralight.org:8443//mnt/hadoop/path/to/file
fdt://.*:[0-9]+/?.*
Aggregates the same source, destination hosts into group transfer.
Same source, destination value is defined by
hostSrc:portSrc-hostDest:postDest
"""
pattern = re.compile("fdt://(?P<host>.*):"
"(?P<port>[0-9]+)/?(?P<file>/.*)")
if not pattern.match(urlSrc):
raise FDTCopyException("Wrong format of '%s'" % urlSrc)
if not pattern.match(urlDest):
raise FDTCopyException("Wrong format of '%s'" % urlDest)
m = pattern.search(urlSrc)
hostSrc, portSrc, fileSrc = (m.group("host"), m.group("port"),
m.group("file"))
m = pattern.search(urlDest)
hostDest, portDest, fileDest = (m.group("host"), m.group("port"),
m.group("file"))
# check if such transfer exists, if so, aggregate, if not, create it
key = "%s:%s-%s:%s" % (hostSrc, portSrc, hostDest, portDest)
try:
transfer = self.transfers[key]
transfer.addFile(TransferFile(fileSrc, fileDest))
except KeyError:
transfer = Transfer(conf, apMon, logger)
trFile = TransferFile(fileSrc, fileDest)
transfer.setUp(hostSrc, portSrc, hostDest, portDest, trFile)
self.transfers[key] = transfer
def call(self, action):
self.logger.debug("Calling '%s' request: %s\n%s ..." %
(self.uri, action.__class__.__name__, action))
try:
try:
if action.timeout:
# raise alarm in timeout seconds
signal.alarm(action.timeout)
result = self.proxy.service(action)
finally:
signal.alarm(0) # disable alarm
except TimeoutException, ex:
m = ("Call to remote PYRO FDTD service timed-out (remote "
"service down, firewall, or ? ...).")
raise FDTCopyException(m)
class Transfers(object):
"""
Wrapper for all transfers, does copyjobfile processing if specified.
"""
def __init__(self, conf, apMon, logger):
# dict of Transfer class instances
# key is hostPortSrc+hostPortDest - processing for
# copyjobfile - transfers grouping
self.transfers = {}
if not conf.get("copyjobfile"):
# consider no copyjobfile scenario - must be a single file
self._setTransfer(conf.get("urlSrc"), conf.get("urlDest"), apMon,
conf, logger)
else:
self._processCopyJobFile(conf, apMon, logger)
def _setTransfer(self, urlSrc, urlDest, apMon, conf, logger):
"""
Initialise instance of Transfer class and set up properties.
URL form:
fdt://gridftp01.ultralight.org:8443//mnt/hadoop/path/to/file
fdt://.*:[0-9]+/?.*
Aggregates the same source, destination hosts into group transfer.
Same source, destination value is defined by
hostSrc:portSrc-hostDest:postDest
"""
pattern = re.compile("fdt://(?P<host>.*):"
"(?P<port>[0-9]+)/?(?P<file>/.*)")
if not pattern.match(urlSrc):
raise FDTCopyException("Wrong format of '%s'" % urlSrc)
if not pattern.match(urlDest):
raise FDTCopyException("Wrong format of '%s'" % urlDest)
m = pattern.search(urlSrc)
hostSrc, portSrc, fileSrc = (m.group("host"), m.group("port"),
m.group("file"))
m = pattern.search(urlDest)
hostDest, portDest, fileDest = (m.group("host"), m.group("port"),
m.group("file"))
# check if such transfer exists, if so, aggregate, if not, create it
key = "%s:%s-%s:%s" % (hostSrc, portSrc, hostDest, portDest)
try:
transfer = self.transfers[key]
transfer.addFile(TransferFile(fileSrc, fileDest))
except KeyError:
transfer = Transfer(conf, apMon, logger)
trFile = TransferFile(fileSrc, fileDest)
transfer.setUp(hostSrc, portSrc, hostDest, portDest, trFile)
self.transfers[key] = transfer
def _processCopyJobFile(self, conf, apMon, logger):
"""
Process copyjobfile - list of pairs urlSrc urlDest
(resp. FROM_PFN TO_PFN).
"""
fileName = conf.get("copyjobfile")
try:
lines = open(fileName, 'r').readlines()
except IOError, ex:
raise FDTCopyException("Can't read '%s', reason: %s" %
(fileName, ex))
try:
for index, line in enumerate(lines):
line = line.strip()
urls = line.split()
urlSrc, urlDest = urls[0], urls[1]
self._setTransfer(urlSrc, urlDest, apMon, conf, logger)
else:
if len(self.transfers) == 0:
raise FDTCopyException("No transfer requests found "
"in '%s'" % fileName)
except IndexError, ex:
# count lines from 1 for user-friendly error message
lineNum = index + 1
m = ("Can't parse file '%s', line: '%s' line number: %s, "
"reason: %s" % (fileName, line, lineNum, ex))
raise FDTCopyException(m)
def _runAuthChain(self):
# 1) contact remote parties in order to obtain port on which
# AuthService runs, then run AuthClient locally to perform
# authentication first with the destination, then with source site
authServiceAction = AuthServiceAction(self.id)
# inconsistent design - catching exceptions / checking result status?
result = self.receiver.perform(authServiceAction)
authServicePortReceiver = result.serverPort
self.logger.info("Remote AuthService (receiver): %s:%s" %
(result.host, authServicePortReceiver))
result = self.sender.perform(authServiceAction)
authServicePortSender = result.serverPort
self.logger.info("Remote AuthService (sender): %s:%s" %
(result.host, authServicePortSender))
# 2) run local AuthClient and authenticate with A, then B
# set options so that AuthClient know which host:port to contact
options = dict(host=self.hostDest, port=authServicePortReceiver)
authClientAction = AuthClientAction(self.id, options)
# execute locally AuthClient and authenticate with destination
# remote site
# (the one which will receive files)
result, remoteGridUserDest = authClientAction.execute(
conf=self.conf, logger=self.logger)
self.logger.debug("Authentication client result: %s, Grid user name "
"at destination: %s" % (result, remoteGridUserDest))
# inconsistent design - catching exceptions / checking result status?
if result.status != 0:
raise FDTCopyException("Authentication failed, reason: %s" %
result.msg)
# if no exception was raised here, first local-"receiver site"
# authentication was successful
# run local AuthClient - source remote site (the one which will
# send files)
# set options so that AuthClient know which host:port to contact
options = dict(host=self.hostSrc, port=authServicePortSender)
authClientAction = AuthClientAction(self.id, options)
# execute locally AuthClient
result, remoteGridUserSrc = authClientAction.execute(
conf=self.conf, logger=self.logger)
self.logger.debug("Authentication client result: %s, Grid user name "
"at source: %s" % (result, remoteGridUserSrc))
# inconsistent design - catching exceptions / checking result status?
if result.status != 0:
raise FDTCopyException("Authentication failed, reason: %s" %
result.msg)
# if no exception was raised, authentication chain was successful
# no need to perform clean up - AuthService processes are running only
# in single instances at remote sites and taken care of by the
# daemon process
self.logger.info("Authentication chain was successful.")
return remoteGridUserSrc, remoteGridUserDest
if action.timeout:
# raise alarm in timeout seconds
signal.alarm(action.timeout)
result = self.proxy.service(action)
finally:
signal.alarm(0) # disable alarm
except TimeoutException, ex:
m = ("Call to remote PYRO FDTD service timed-out (remote "
"service down, firewall, or ? ...).")
raise FDTCopyException(m)
# ProtocolError first, before PyroError
except ProtocolError, ex:
m = "ProtocolError during remote PYRO call, reason: %s" % ex
self.logger.error("PYRO call traceback: %s" %
''.join(Pyro.util.getPyroTraceback(ex)))
raise FDTCopyException(m)
except PyroError, ex:
self.logger.error("PYRO call traceback: %s" %
''.join(Pyro.util.getPyroTraceback(ex)))
m = "PyroError during remote PYRO call, reason: %s" % ex
raise FDTCopyException(m)
except ConnectionClosedError, ex:
m = "Connection with remote PYRO service lost, try again."
raise FDTCopyException(m)
else:
#self.logger.debug("Call to remote PYRO service successful.")
return result
def perform(self, action):
result = self.call(action)
if result.status == 0: