def __init__(self, user, config): """ store the user and tfc the worker """ self.user = user self.config = config self.dropbox_dir = '%s/dropbox/inputs' % self.config.componentDir logging.basicConfig(level=config.log_level) self.site_tfc_map = {} self.logger = logging.getLogger('AsyncTransfer-Reporter-%s' % self.user) formatter = getCommonLogFormatter(self.config) for handler in logging.getLogger().handlers: handler.setFormatter(formatter) self.uiSetupScript = getattr(self.config, 'UISetupScript', None) self.cleanEnvironment = '' self.userDN = '' self.init = True if getattr(self.config, 'cleanEnvironment', False): self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;' # TODO: improve how the worker gets a log self.logger.debug("Trying to get DN") try: self.userDN = getDNFromUserName(self.user, self.logger) except Exception as ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) self.init = False return if not self.userDN: self.init = False return defaultDelegation = { 'logger': self.logger, 'credServerPath': self.config.credentialDir, # It will be moved to be getfrom couchDB 'myProxySvr': 'myproxy.cern.ch', 'min_time_left' : getattr(self.config, 'minTimeLeft', 36000), 'serverDN': self.config.serverDN, 'uisource': self.uiSetupScript, 'cleanEnvironment': getattr(self.config, 'cleanEnvironment', False) } if hasattr(self.config, "cache_area"): try: defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.config.cache_area)[0] except IndexError: self.logger.error('MyproxyAccount parameter cannot be retrieved from %s' % self.config.cache_area) pass if getattr(self.config, 'serviceCert', None): defaultDelegation['server_cert'] = self.config.serviceCert if getattr(self.config, 'serviceKey', None): defaultDelegation['server_key'] = self.config.serviceKey self.valid = False try: self.valid, proxy = getProxy(self.userDN, "", "", defaultDelegation, self.logger) except Exception as ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) if self.valid: self.userProxy = proxy else: # Use the operator's proxy when the user proxy in invalid. # This will be moved soon self.logger.error('Did not get valid proxy. Setting proxy to ops proxy') self.userProxy = config.opsProxy if self.config.isOracle: try: self.oracleDB = HTTPRequests(self.config.oracleDB, config.opsProxy, config.opsProxy) except Exception: self.logger.exception() raise else: server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir) self.commandTimeout = 1200 self.max_retry = config.max_retry # Proxy management in Couch os.environ['X509_USER_PROXY'] = self.userProxy try: self.phedex = PhEDEx(responseType='xml', dict={'key':self.config.opsProxy, 'cert':self.config.opsProxy}) except Exception as e: self.logger.exception('PhEDEx exception: %s' % e)
def killThread(self, thread_id, transfers): """This is the worker thread function for kill command. """ while True: transfer_list = transfers.get() self.logger.info("Starting thread %s" % (thread_id)) user = transfer_list[0]['username'] group = transfer_list[0]['user_group'] role = transfer_list[0]['user_role'] uiSetupScript = getattr(self.config, 'UISetupScript', None) self.logger.debug("Trying to get DN for %s %s %s %s" % (user, self.logger, self.config.opsProxy, self.config.opsProxy)) try: userDN = getDNFromUserName(user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy) except Exception as ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue if not userDN: transfers.task_done() time.sleep(1) continue self.logger.debug("user DN: %s" % userDN) try: defaultDelegation = {'logger': self.logger, 'credServerPath': self.config.credentialDir, 'myProxySvr': 'myproxy.cern.ch', 'min_time_left': getattr(self.config, 'minTimeLeft', 36000), 'serverDN': self.config.serverDN, 'uisource': uiSetupScript, 'cleanEnvironment': getattr(self.config, 'cleanEnvironment', False)} if hasattr(self.config, "cache_area"): cache_area = self.config.cache_area defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(cache_area)[0] except IndexError: self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % self.config.cache_area) transfers.task_done() time.sleep(1) continue if getattr(self.config, 'serviceCert', None): defaultDelegation['server_cert'] = self.config.serviceCert if getattr(self.config, 'serviceKey', None): defaultDelegation['server_key'] = self.config.serviceKey try: defaultDelegation['userDN'] = userDN defaultDelegation['group'] = group if group else '' defaultDelegation['role'] = role if group else '' self.logger.debug('delegation: %s' % defaultDelegation) valid_proxy, user_proxy = getProxy(defaultDelegation, self.logger) except Exception as ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) transfers.task_done() time.sleep(1) continue # TODO: take server from db, right now, take only the first of the list and assuming it valid for all try: # TODO: debug u added during info upload. To be fixed soon! For now worked around fts_server = transfer_list[0]['fts_instance'].split('u')[1] self.logger.info("Delegating proxy to %s" % fts_server) context = fts3.Context(fts_server, user_proxy, user_proxy, verify=True) self.logger.debug(fts3.delegate(context, lifetime=timedelta(hours=48), force=False)) self.logger.info("Proxy delegated. Grouping files by jobId") jobs = {} for fileToKill in transfer_list: # TODO: debug u added during info upload. To be fixed soon! For now worked around jid = str(fileToKill['fts_id']).split('u')[1] if jid not in jobs: jobs[jid] = [] jobs[jid].append(fileToKill) self.logger.info("Found %s jobIds", len(jobs.keys())) self.logger.debug("jobIds: %s", jobs.keys) # list for files killed or failed to killed = [] too_late = [] for ftsJobId, files in jobs.iteritems(): self.logger.info("Cancelling tranfers in %s" % ftsJobId) ref_lfns = [str(x['destination_lfn'].split('/store/')[1]) for x in files] source_lfns = [x['source_lfn'] for x in files] job_list = fts3.get_job_status(context, ftsJobId, list_files=True) tx = job_list['files'] # TODO: this workaround is needed to get FTS file id, we may want to add a column in the db? idListToKill = [x['file_id'] for x in tx if x['dest_surl'].split('/cms/store/')[1] in ref_lfns] # needed for the state update lfnListToKill = [ref_lfns.index(str(x['dest_surl'].split('/cms/store/')[1])) for x in tx if x['dest_surl'].split('/cms/store/')[1] in ref_lfns] self.logger.debug("List of ids to cancel for job %s: %s" % (ftsJobId, idListToKill)) res = fts3.cancel(context, ftsJobId, idListToKill) self.logger.debug('Kill command result: %s' % json.dumps(res)) if not isinstance(res, list): res = [res] # Verify if the kill command succeeded for k, kill_res in enumerate(res): indexToUpdate = lfnListToKill[k] if kill_res in ("FINISHEDDIRTY", "FINISHED", "FAILED"): self.logger.debug(source_lfns[indexToUpdate]) too_late.append(getHashLfn(source_lfns[indexToUpdate])) else: killed.append(getHashLfn(source_lfns[indexToUpdate])) # TODO: decide how to update status for too_late files killed += too_late self.logger.debug('Updating status of killed files: %s' % killed) if len(killed) > 0: data = dict() data['asoworker'] = self.config.asoworker data['subresource'] = 'updateTransfers' data['list_of_ids'] = killed data['list_of_transfer_state'] = ["KILLED" for _ in killed] self.oracleDB.post(self.config.oracleFileTrans, data=encodeRequest(data)) self.logger.debug("Marked killed %s" % killed) except: # TODO: split and improve try/except self.logger.exception('Kill command failed') transfers.task_done()
def __init__(self, user, tfc_map, config): """ store the user and tfc the worker """ self.user = user[0] self.group = user[1] self.role = user[2] self.tfc_map = tfc_map self.config = config self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir logging.basicConfig(level=config.log_level) self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user) formatter = getCommonLogFormatter(self.config) for handler in logging.getLogger().handlers: handler.setFormatter(formatter) self.pfn_to_lfn_mapping = {} self.max_retry = config.max_retry self.uiSetupScript = getattr(self.config, 'UISetupScript', None) self.submission_command = getattr(self.config, 'submission_command', 'glite-transfer-submit') self.cleanEnvironment = '' self.userDN = '' self.init = True if getattr(self.config, 'cleanEnvironment', False): self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;' self.logger.debug("Trying to get DN for %s" % self.user) try: self.userDN = getDNFromUserName(self.user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy) except Exception as ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) self.init = False return if not self.userDN: self.init = False return defaultDelegation = {'logger': self.logger, 'credServerPath' : self.config.credentialDir, # It will be moved to be getfrom couchDB 'myProxySvr': 'myproxy.cern.ch', 'min_time_left' : getattr(self.config, 'minTimeLeft', 36000), 'serverDN' : self.config.serverDN, 'uisource' : self.uiSetupScript, 'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False)} # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace=self.config.pluginDir) self.commandTimeout = 1200 server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.config_db = config_server.connectDatabase(self.config.config_database) self.fts_server_for_transfer = getFTServer("T1_UK_RAL", 'getRunningFTSserver', self.config_db, self.logger) self.cache_area="" if hasattr(self.config, "cache_area"): self.cache_area = self.config.cache_area query = {'key':self.user} try: self.user_cache_area = self.db.loadView('DBSPublisher', 'cache_area', query)['rows'] self.cache_area = "https://"+self.user_cache_area[0]['value'][0]+self.user_cache_area[0]['value'][1]+"/filemetadata" except Exception as ex: msg = "Error getting user cache_area." msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) pass try: defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.cache_area)[0] except IndexError: self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % (self.config.cache_area)) if getattr(self.config, 'serviceCert', None): defaultDelegation['server_cert'] = self.config.serviceCert if getattr(self.config, 'serviceKey', None): defaultDelegation['server_key'] = self.config.serviceKey self.valid_proxy = False self.user_proxy = None try: defaultDelegation['userDN'] = self.userDN defaultDelegation['group'] = self.group defaultDelegation['role'] = self.role self.valid_proxy, self.user_proxy = getProxy(defaultDelegation, self.logger) except Exception as ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg)
'A problem occured when contacting couchDB to update last cleanning time: %s' % e) return self.logger.info('LFNs to remove: %s' % len(all_LFNs)) self.logger.debug('LFNs to remove: %s' % all_LFNs) lfn_to_proxy = {} for lfnDetails in all_LFNs: lfn = lfnDetails['value']['lfn'] user = lfn.split('/')[4].split('.')[0] if not lfn_to_proxy.has_key(user): valid_proxy = False try: userDN = getDNFromUserName(user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy) valid_proxy, userProxy = getProxy( userDN, "", "", self.defaultDelegation, self.logger) except Exception, ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) if valid_proxy: lfn_to_proxy[user] = userProxy else: lfn_to_proxy[user] = self.opsProxy userProxy = lfn_to_proxy[user] location = lfnDetails['value']['location']
self.config_db.makeRequest(uri = updateUri, type = "PUT", decode = False) except Exception, e: self.logger.exception('A problem occured when contacting couchDB to update last cleanning time: %s' % e) return self.logger.info('LFNs to remove: %s' % len(all_LFNs)) self.logger.debug('LFNs to remove: %s' % all_LFNs) lfn_to_proxy = {} for lfnDetails in all_LFNs: lfn = lfnDetails['value']['lfn'] user = lfn.split('/')[4].split('.')[0] if not lfn_to_proxy.has_key(user): valid_proxy = False try: userDN = getDNFromUserName(user, self.logger, ckey = self.config.opsProxy, cert = self.config.opsProxy) valid_proxy, userProxy = getProxy(userDN, "", "", self.defaultDelegation, self.logger) except Exception, ex: msg = "Error getting the user proxy" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) if valid_proxy: lfn_to_proxy[user] = userProxy else: lfn_to_proxy[user] = self.opsProxy userProxy = lfn_to_proxy[user] location = lfnDetails['value']['location'] self.logger.info("Removing %s from %s" %(lfn, location)) pfn = self.apply_tfc_to_lfn( '%s:%s' %(location, lfn)) if pfn: