def editLogger(self):
#logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
#self.logger = logging.getLogger()
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
formatter = getCommonLogFormatter(self.config)
for handler in self.logger.handlers:
handler.setFormatter(formatter)
def editLogger(self):
#logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
#self.logger = logging.getLogger()
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
formatter = getCommonLogFormatter(self.config)
for handler in self.logger.handlers:
handler.setFormatter(formatter)
def __init__(self, user, config):
"""
store the user and tfc the worker
"""
self.user = user
self.config = config
self.dropbox_dir = '%s/dropbox/inputs' % self.config.componentDir
logging.basicConfig(level=config.log_level)
self.site_tfc_map = {}
self.logger = logging.getLogger('AsyncTransfer-Reporter-%s' % self.user)
formatter = getCommonLogFormatter(self.config)
for handler in logging.getLogger().handlers:
handler.setFormatter(formatter)
self.uiSetupScript = getattr(self.config, 'UISetupScript', None)
self.cleanEnvironment = ''
self.userDN = ''
self.init = True
if getattr(self.config, 'cleanEnvironment', False):
self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;'
# TODO: improve how the worker gets a log
self.logger.debug("Trying to get DN")
try:
self.userDN = getDNFromUserName(self.user, self.logger)
except Exception as ex:
msg = "Error retrieving the user DN"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
self.init = False
return
if not self.userDN:
self.init = False
return
defaultDelegation = {
'logger': self.logger,
'credServerPath': self.config.credentialDir,
# It will be moved to be getfrom couchDB
'myProxySvr': 'myproxy.cern.ch',
'min_time_left' : getattr(self.config, 'minTimeLeft', 36000),
'serverDN': self.config.serverDN,
'uisource': self.uiSetupScript,
'cleanEnvironment': getattr(self.config, 'cleanEnvironment', False)
}
if hasattr(self.config, "cache_area"):
try:
defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.config.cache_area)[0]
except IndexError:
self.logger.error('MyproxyAccount parameter cannot be retrieved from %s' % self.config.cache_area)
pass
if getattr(self.config, 'serviceCert', None):
defaultDelegation['server_cert'] = self.config.serviceCert
if getattr(self.config, 'serviceKey', None):
defaultDelegation['server_key'] = self.config.serviceKey
self.valid = False
try:
self.valid, proxy = getProxy(self.userDN, "", "", defaultDelegation, self.logger)
except Exception as ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
if self.valid:
self.userProxy = proxy
else:
# Use the operator's proxy when the user proxy in invalid.
# This will be moved soon
self.logger.error('Did not get valid proxy. Setting proxy to ops proxy')
self.userProxy = config.opsProxy
if self.config.isOracle:
try:
self.oracleDB = HTTPRequests(self.config.oracleDB,
config.opsProxy,
config.opsProxy)
except Exception:
self.logger.exception()
raise
else:
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
self.commandTimeout = 1200
self.max_retry = config.max_retry
# Proxy management in Couch
os.environ['X509_USER_PROXY'] = self.userProxy
try:
self.phedex = PhEDEx(responseType='xml',
dict={'key':self.config.opsProxy,
'cert':self.config.opsProxy})
except Exception as e:
self.logger.exception('PhEDEx exception: %s' % e)
def __init__(self, user, tfc_map, config):
"""
store the user and tfc the worker
"""
self.user = user[0]
self.group = user[1]
self.role = user[2]
self.tfc_map = tfc_map
self.config = config
self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir
logging.basicConfig(level=config.log_level)
self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user)
formatter = getCommonLogFormatter(self.config)
for handler in logging.getLogger().handlers:
handler.setFormatter(formatter)
self.pfn_to_lfn_mapping = {}
self.max_retry = config.max_retry
self.uiSetupScript = getattr(self.config, 'UISetupScript', None)
self.submission_command = getattr(self.config, 'submission_command', 'glite-transfer-submit')
self.cleanEnvironment = ''
self.userDN = ''
self.init = True
if getattr(self.config, 'cleanEnvironment', False):
self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;'
self.logger.debug("Trying to get DN for %s" % self.user)
try:
self.userDN = getDNFromUserName(self.user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy)
except Exception as ex:
msg = "Error retrieving the user DN"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
self.init = False
return
if not self.userDN:
self.init = False
return
defaultDelegation = {'logger': self.logger,
'credServerPath' : self.config.credentialDir,
# It will be moved to be getfrom couchDB
'myProxySvr': 'myproxy.cern.ch',
'min_time_left' : getattr(self.config, 'minTimeLeft', 36000),
'serverDN' : self.config.serverDN,
'uisource' : self.uiSetupScript,
'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False)}
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace=self.config.pluginDir)
self.commandTimeout = 1200
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(self.config.config_database)
self.fts_server_for_transfer = getFTServer("T1_UK_RAL", 'getRunningFTSserver', self.config_db, self.logger)
self.cache_area=""
if hasattr(self.config, "cache_area"):
self.cache_area = self.config.cache_area
query = {'key':self.user}
try:
self.user_cache_area = self.db.loadView('DBSPublisher', 'cache_area', query)['rows']
self.cache_area = "https://"+self.user_cache_area[0]['value'][0]+self.user_cache_area[0]['value'][1]+"/filemetadata"
except Exception as ex:
msg = "Error getting user cache_area."
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
pass
try:
defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.cache_area)[0]
except IndexError:
self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % (self.config.cache_area))
if getattr(self.config, 'serviceCert', None):
defaultDelegation['server_cert'] = self.config.serviceCert
if getattr(self.config, 'serviceKey', None):
defaultDelegation['server_key'] = self.config.serviceKey
self.valid_proxy = False
self.user_proxy = None
try:
defaultDelegation['userDN'] = self.userDN
defaultDelegation['group'] = self.group
defaultDelegation['role'] = self.role
self.valid_proxy, self.user_proxy = getProxy(defaultDelegation, self.logger)
except Exception as ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
