def updateStat(self, document): """ Update the stat documents list. """ ftserver = getFTServer(document['destination'], 'getAllFTSserver', self.config_db, self.logger) for oldDoc in self.iteration_docs: if oldDoc['fts'] == ftserver: self.updateServerDocument(oldDoc, document) return self.createServerDocument(ftserver, document) return
def __init__(self, user, tfc_map, config): """ store the user and tfc the worker """ self.user = user[0] self.group = user[1] self.role = user[2] self.tfc_map = tfc_map self.config = config self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir logging.basicConfig(level=config.log_level) self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user) formatter = getCommonLogFormatter(self.config) for handler in logging.getLogger().handlers: handler.setFormatter(formatter) self.pfn_to_lfn_mapping = {} self.max_retry = config.max_retry self.uiSetupScript = getattr(self.config, 'UISetupScript', None) self.submission_command = getattr(self.config, 'submission_command', 'glite-transfer-submit') self.cleanEnvironment = '' self.userDN = '' self.init = True if getattr(self.config, 'cleanEnvironment', False): self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;' self.logger.debug("Trying to get DN for %s" % self.user) try: self.userDN = getDNFromUserName(self.user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy) except Exception as ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) self.init = False return if not self.userDN: self.init = False return defaultDelegation = {'logger': self.logger, 'credServerPath' : self.config.credentialDir, # It will be moved to be getfrom couchDB 'myProxySvr': 'myproxy.cern.ch', 'min_time_left' : getattr(self.config, 'minTimeLeft', 36000), 'serverDN' : self.config.serverDN, 'uisource' : self.uiSetupScript, 'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False)} # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace=self.config.pluginDir) self.commandTimeout = 1200 server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.config_db = config_server.connectDatabase(self.config.config_database) self.fts_server_for_transfer = getFTServer("T1_UK_RAL", 'getRunningFTSserver', self.config_db, self.logger) self.cache_area="" if hasattr(self.config, "cache_area"): self.cache_area = self.config.cache_area query = {'key':self.user} try: self.user_cache_area = self.db.loadView('DBSPublisher', 'cache_area', query)['rows'] self.cache_area = "https://"+self.user_cache_area[0]['value'][0]+self.user_cache_area[0]['value'][1]+"/filemetadata" except Exception as ex: msg = "Error getting user cache_area." msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) pass try: defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.cache_area)[0] except IndexError: self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % (self.config.cache_area)) if getattr(self.config, 'serviceCert', None): defaultDelegation['server_cert'] = self.config.serviceCert if getattr(self.config, 'serviceKey', None): defaultDelegation['server_key'] = self.config.serviceKey self.valid_proxy = False self.user_proxy = None try: defaultDelegation['userDN'] = self.userDN defaultDelegation['group'] = self.group defaultDelegation['role'] = self.role self.valid_proxy, self.user_proxy = getProxy(defaultDelegation, self.logger) except Exception as ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg)
class TransferWorker: def __init__(self, user, tfc_map, config): """ store the user and tfc the worker """ self.user = user[0] self.group = user[1] self.role = user[2] self.tfc_map = tfc_map self.config = config self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir logging.basicConfig(level=config.log_level) self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user) self.pfn_to_lfn_mapping = {} self.max_retry = config.max_retry self.uiSetupScript = getattr(self.config, 'UISetupScript', None) self.submission_command = getattr(self.config, 'submission_command', 'glite-transfer-submit') self.cleanEnvironment = '' self.userDN = '' self.init = True if getattr(self.config, 'cleanEnvironment', False): self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;' self.logger.debug("Trying to get DN for %s" % self.user) try: self.userDN = "/C=IT/O=INFN/OU=Personal Certificate/L=Perugia/CN=Diego Ciangottini" #getDNFromUserName(self.user, self.logger) except Exception, ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) self.init = False return if not self.userDN: self.init = False return defaultDelegation = { 'logger': self.logger, 'credServerPath' : \ self.config.credentialDir, # It will be moved to be getfrom couchDB 'myProxySvr': 'myproxy.cern.ch', 'min_time_left' : getattr(self.config, 'minTimeLeft', 36000), 'serverDN' : self.config.serverDN, 'uisource' : self.uiSetupScript, 'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False) } if hasattr(self.config, "cache_area"): try: defaultDelegation['myproxyAccount'] = re.compile( 'https?://([^/]*)/.*').findall(self.config.cache_area)[0] except IndexError: self.logger.error( 'MyproxyAccount parameter cannot be retrieved from %s' % self.config.cache_area) pass if getattr(self.config, 'serviceCert', None): defaultDelegation['server_cert'] = self.config.serviceCert if getattr(self.config, 'serviceKey', None): defaultDelegation['server_key'] = self.config.serviceKey self.valid = False # try: # self.valid, proxy = getProxy(self.userDN, self.group, self.role, defaultDelegation, self.logger) # except Exception, ex: # msg = "Error getting the user proxy" # msg += str(ex) # msg += str(traceback.format_exc()) # self.logger.error(msg) #self.valid = 0 #if self.valid: # self.userProxy = proxy #else: # Use the operator's proxy when the user proxy in invalid. # This will be moved soon #self.logger.error('Did not get valid proxy. Setting proxy to ops proxy') self.userProxy = config.opsProxy # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace=self.config.pluginDir) self.failures_reasons = {} self.commandTimeout = 1200 self.polling_cycle = 600 os.environ['X509_USER_PROXY'] = self.userProxy server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.config_db = config_server.connectDatabase( self.config.config_database) self.fts_server_for_transfer = getFTServer("T1_UK_RAL", 'getRunningFTSserver', self.config_db, self.logger)