def editLogger(self): #logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M') #self.logger = logging.getLogger() # self.logger is set up by the BaseWorkerThread, we just set it's level try: self.logger.setLevel(self.config.log_level) except: self.logger = logging.getLogger() self.logger.setLevel(self.config.log_level) formatter = getCommonLogFormatter(self.config) for handler in self.logger.handlers: handler.setFormatter(formatter)
def __init__(self, user, config): """ store the user and tfc the worker """ self.user = user self.config = config self.dropbox_dir = '%s/dropbox/inputs' % self.config.componentDir logging.basicConfig(level=config.log_level) self.site_tfc_map = {} self.logger = logging.getLogger('AsyncTransfer-Reporter-%s' % self.user) formatter = getCommonLogFormatter(self.config) for handler in logging.getLogger().handlers: handler.setFormatter(formatter) self.uiSetupScript = getattr(self.config, 'UISetupScript', None) self.cleanEnvironment = '' self.userDN = '' self.init = True if getattr(self.config, 'cleanEnvironment', False): self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;' # TODO: improve how the worker gets a log self.logger.debug("Trying to get DN") try: self.userDN = getDNFromUserName(self.user, self.logger) except Exception as ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) self.init = False return if not self.userDN: self.init = False return defaultDelegation = { 'logger': self.logger, 'credServerPath': self.config.credentialDir, # It will be moved to be getfrom couchDB 'myProxySvr': 'myproxy.cern.ch', 'min_time_left' : getattr(self.config, 'minTimeLeft', 36000), 'serverDN': self.config.serverDN, 'uisource': self.uiSetupScript, 'cleanEnvironment': getattr(self.config, 'cleanEnvironment', False) } if hasattr(self.config, "cache_area"): try: defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.config.cache_area)[0] except IndexError: self.logger.error('MyproxyAccount parameter cannot be retrieved from %s' % self.config.cache_area) pass if getattr(self.config, 'serviceCert', None): defaultDelegation['server_cert'] = self.config.serviceCert if getattr(self.config, 'serviceKey', None): defaultDelegation['server_key'] = self.config.serviceKey self.valid = False try: self.valid, proxy = getProxy(self.userDN, "", "", defaultDelegation, self.logger) except Exception as ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) if self.valid: self.userProxy = proxy else: # Use the operator's proxy when the user proxy in invalid. # This will be moved soon self.logger.error('Did not get valid proxy. Setting proxy to ops proxy') self.userProxy = config.opsProxy if self.config.isOracle: try: self.oracleDB = HTTPRequests(self.config.oracleDB, config.opsProxy, config.opsProxy) except Exception: self.logger.exception() raise else: server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir) self.commandTimeout = 1200 self.max_retry = config.max_retry # Proxy management in Couch os.environ['X509_USER_PROXY'] = self.userProxy try: self.phedex = PhEDEx(responseType='xml', dict={'key':self.config.opsProxy, 'cert':self.config.opsProxy}) except Exception as e: self.logger.exception('PhEDEx exception: %s' % e)
def __init__(self, user, tfc_map, config): """ store the user and tfc the worker """ self.user = user[0] self.group = user[1] self.role = user[2] self.tfc_map = tfc_map self.config = config self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir logging.basicConfig(level=config.log_level) self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user) formatter = getCommonLogFormatter(self.config) for handler in logging.getLogger().handlers: handler.setFormatter(formatter) self.pfn_to_lfn_mapping = {} self.max_retry = config.max_retry self.uiSetupScript = getattr(self.config, 'UISetupScript', None) self.submission_command = getattr(self.config, 'submission_command', 'glite-transfer-submit') self.cleanEnvironment = '' self.userDN = '' self.init = True if getattr(self.config, 'cleanEnvironment', False): self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;' self.logger.debug("Trying to get DN for %s" % self.user) try: self.userDN = getDNFromUserName(self.user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy) except Exception as ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) self.init = False return if not self.userDN: self.init = False return defaultDelegation = {'logger': self.logger, 'credServerPath' : self.config.credentialDir, # It will be moved to be getfrom couchDB 'myProxySvr': 'myproxy.cern.ch', 'min_time_left' : getattr(self.config, 'minTimeLeft', 36000), 'serverDN' : self.config.serverDN, 'uisource' : self.uiSetupScript, 'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False)} # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace=self.config.pluginDir) self.commandTimeout = 1200 server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.config_db = config_server.connectDatabase(self.config.config_database) self.fts_server_for_transfer = getFTServer("T1_UK_RAL", 'getRunningFTSserver', self.config_db, self.logger) self.cache_area="" if hasattr(self.config, "cache_area"): self.cache_area = self.config.cache_area query = {'key':self.user} try: self.user_cache_area = self.db.loadView('DBSPublisher', 'cache_area', query)['rows'] self.cache_area = "https://"+self.user_cache_area[0]['value'][0]+self.user_cache_area[0]['value'][1]+"/filemetadata" except Exception as ex: msg = "Error getting user cache_area." msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) pass try: defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.cache_area)[0] except IndexError: self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % (self.config.cache_area)) if getattr(self.config, 'serviceCert', None): defaultDelegation['server_cert'] = self.config.serviceCert if getattr(self.config, 'serviceKey', None): defaultDelegation['server_key'] = self.config.serviceKey self.valid_proxy = False self.user_proxy = None try: defaultDelegation['userDN'] = self.userDN defaultDelegation['group'] = self.group defaultDelegation['role'] = self.role self.valid_proxy, self.user_proxy = getProxy(defaultDelegation, self.logger) except Exception as ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg)