def updateStat(self, document):
"""
Update the stat documents list.
"""
ftserver = getFTServer(document['destination'], 'getAllFTSserver', self.config_db, self.logger)
for oldDoc in self.iteration_docs:
if oldDoc['fts'] == ftserver:
self.updateServerDocument(oldDoc, document)
return
self.createServerDocument(ftserver, document)
return
def updateStat(self, document):
"""
Update the stat documents list.
"""
ftserver = getFTServer(document['destination'], 'getAllFTSserver', self.config_db, self.logger)
for oldDoc in self.iteration_docs:
if oldDoc['fts'] == ftserver:
self.updateServerDocument(oldDoc, document)
return
self.createServerDocument(ftserver, document)
return
def __init__(self, user, tfc_map, config):
"""
store the user and tfc the worker
"""
self.user = user[0]
self.group = user[1]
self.role = user[2]
self.tfc_map = tfc_map
self.config = config
self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir
logging.basicConfig(level=config.log_level)
self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user)
formatter = getCommonLogFormatter(self.config)
for handler in logging.getLogger().handlers:
handler.setFormatter(formatter)
self.pfn_to_lfn_mapping = {}
self.max_retry = config.max_retry
self.uiSetupScript = getattr(self.config, 'UISetupScript', None)
self.submission_command = getattr(self.config, 'submission_command', 'glite-transfer-submit')
self.cleanEnvironment = ''
self.userDN = ''
self.init = True
if getattr(self.config, 'cleanEnvironment', False):
self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;'
self.logger.debug("Trying to get DN for %s" % self.user)
try:
self.userDN = getDNFromUserName(self.user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy)
except Exception as ex:
msg = "Error retrieving the user DN"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
self.init = False
return
if not self.userDN:
self.init = False
return
defaultDelegation = {'logger': self.logger,
'credServerPath' : self.config.credentialDir,
# It will be moved to be getfrom couchDB
'myProxySvr': 'myproxy.cern.ch',
'min_time_left' : getattr(self.config, 'minTimeLeft', 36000),
'serverDN' : self.config.serverDN,
'uisource' : self.uiSetupScript,
'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False)}
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace=self.config.pluginDir)
self.commandTimeout = 1200
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(self.config.config_database)
self.fts_server_for_transfer = getFTServer("T1_UK_RAL", 'getRunningFTSserver', self.config_db, self.logger)
self.cache_area=""
if hasattr(self.config, "cache_area"):
self.cache_area = self.config.cache_area
query = {'key':self.user}
try:
self.user_cache_area = self.db.loadView('DBSPublisher', 'cache_area', query)['rows']
self.cache_area = "https://"+self.user_cache_area[0]['value'][0]+self.user_cache_area[0]['value'][1]+"/filemetadata"
except Exception as ex:
msg = "Error getting user cache_area."
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
pass
try:
defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.cache_area)[0]
except IndexError:
self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % (self.config.cache_area))
if getattr(self.config, 'serviceCert', None):
defaultDelegation['server_cert'] = self.config.serviceCert
if getattr(self.config, 'serviceKey', None):
defaultDelegation['server_key'] = self.config.serviceKey
self.valid_proxy = False
self.user_proxy = None
try:
defaultDelegation['userDN'] = self.userDN
defaultDelegation['group'] = self.group
defaultDelegation['role'] = self.role
self.valid_proxy, self.user_proxy = getProxy(defaultDelegation, self.logger)
except Exception as ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
class TransferWorker:
def __init__(self, user, tfc_map, config):
"""
store the user and tfc the worker
"""
self.user = user[0]
self.group = user[1]
self.role = user[2]
self.tfc_map = tfc_map
self.config = config
self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir
logging.basicConfig(level=config.log_level)
self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user)
self.pfn_to_lfn_mapping = {}
self.max_retry = config.max_retry
self.uiSetupScript = getattr(self.config, 'UISetupScript', None)
self.submission_command = getattr(self.config, 'submission_command',
'glite-transfer-submit')
self.cleanEnvironment = ''
self.userDN = ''
self.init = True
if getattr(self.config, 'cleanEnvironment', False):
self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;'
self.logger.debug("Trying to get DN for %s" % self.user)
try:
self.userDN = "/C=IT/O=INFN/OU=Personal Certificate/L=Perugia/CN=Diego Ciangottini" #getDNFromUserName(self.user, self.logger)
except Exception, ex:
msg = "Error retrieving the user DN"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
self.init = False
return
if not self.userDN:
self.init = False
return
defaultDelegation = {
'logger': self.logger,
'credServerPath' : \
self.config.credentialDir,
# It will be moved to be getfrom couchDB
'myProxySvr': 'myproxy.cern.ch',
'min_time_left' : getattr(self.config, 'minTimeLeft', 36000),
'serverDN' : self.config.serverDN,
'uisource' : self.uiSetupScript,
'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False)
}
if hasattr(self.config, "cache_area"):
try:
defaultDelegation['myproxyAccount'] = re.compile(
'https?://([^/]*)/.*').findall(self.config.cache_area)[0]
except IndexError:
self.logger.error(
'MyproxyAccount parameter cannot be retrieved from %s' %
self.config.cache_area)
pass
if getattr(self.config, 'serviceCert', None):
defaultDelegation['server_cert'] = self.config.serviceCert
if getattr(self.config, 'serviceKey', None):
defaultDelegation['server_key'] = self.config.serviceKey
self.valid = False
# try:
# self.valid, proxy = getProxy(self.userDN, self.group, self.role, defaultDelegation, self.logger)
# except Exception, ex:
# msg = "Error getting the user proxy"
# msg += str(ex)
# msg += str(traceback.format_exc())
# self.logger.error(msg)
#self.valid = 0
#if self.valid:
# self.userProxy = proxy
#else:
# Use the operator's proxy when the user proxy in invalid.
# This will be moved soon
#self.logger.error('Did not get valid proxy. Setting proxy to ops proxy')
self.userProxy = config.opsProxy
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir,
namespace=self.config.pluginDir)
self.failures_reasons = {}
self.commandTimeout = 1200
self.polling_cycle = 600
os.environ['X509_USER_PROXY'] = self.userProxy
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.fts_server_for_transfer = getFTServer("T1_UK_RAL",
'getRunningFTSserver',
self.config_db, self.logger)
