def __init__(self, user, config):
"""
store the user and tfc the worker
"""
self.user = user
self.config = config
self.dropbox_dir = '%s/dropbox/inputs' % self.config.componentDir
logging.basicConfig(level=config.log_level)
self.site_tfc_map = {}
self.logger = logging.getLogger('AsyncTransfer-Reporter-%s' % self.user)
formatter = getCommonLogFormatter(self.config)
for handler in logging.getLogger().handlers:
handler.setFormatter(formatter)
self.uiSetupScript = getattr(self.config, 'UISetupScript', None)
self.cleanEnvironment = ''
self.userDN = ''
self.init = True
if getattr(self.config, 'cleanEnvironment', False):
self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;'
# TODO: improve how the worker gets a log
self.logger.debug("Trying to get DN")
try:
self.userDN = getDNFromUserName(self.user, self.logger)
except Exception as ex:
msg = "Error retrieving the user DN"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
self.init = False
return
if not self.userDN:
self.init = False
return
defaultDelegation = {
'logger': self.logger,
'credServerPath': self.config.credentialDir,
# It will be moved to be getfrom couchDB
'myProxySvr': 'myproxy.cern.ch',
'min_time_left' : getattr(self.config, 'minTimeLeft', 36000),
'serverDN': self.config.serverDN,
'uisource': self.uiSetupScript,
'cleanEnvironment': getattr(self.config, 'cleanEnvironment', False)
}
if hasattr(self.config, "cache_area"):
try:
defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.config.cache_area)[0]
except IndexError:
self.logger.error('MyproxyAccount parameter cannot be retrieved from %s' % self.config.cache_area)
pass
if getattr(self.config, 'serviceCert', None):
defaultDelegation['server_cert'] = self.config.serviceCert
if getattr(self.config, 'serviceKey', None):
defaultDelegation['server_key'] = self.config.serviceKey
self.valid = False
try:
self.valid, proxy = getProxy(self.userDN, "", "", defaultDelegation, self.logger)
except Exception as ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
if self.valid:
self.userProxy = proxy
else:
# Use the operator's proxy when the user proxy in invalid.
# This will be moved soon
self.logger.error('Did not get valid proxy. Setting proxy to ops proxy')
self.userProxy = config.opsProxy
if self.config.isOracle:
try:
self.oracleDB = HTTPRequests(self.config.oracleDB,
config.opsProxy,
config.opsProxy)
except Exception:
self.logger.exception()
raise
else:
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
self.commandTimeout = 1200
self.max_retry = config.max_retry
# Proxy management in Couch
os.environ['X509_USER_PROXY'] = self.userProxy
try:
self.phedex = PhEDEx(responseType='xml',
dict={'key':self.config.opsProxy,
'cert':self.config.opsProxy})
except Exception as e:
self.logger.exception('PhEDEx exception: %s' % e)
def killThread(self, thread_id, transfers):
"""This is the worker thread function for kill command.
"""
while True:
transfer_list = transfers.get()
self.logger.info("Starting thread %s" % (thread_id))
user = transfer_list[0]['username']
group = transfer_list[0]['user_group']
role = transfer_list[0]['user_role']
uiSetupScript = getattr(self.config, 'UISetupScript', None)
self.logger.debug("Trying to get DN for %s %s %s %s" % (user, self.logger, self.config.opsProxy, self.config.opsProxy))
try:
userDN = getDNFromUserName(user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy)
except Exception as ex:
msg = "Error retrieving the user DN"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
continue
if not userDN:
transfers.task_done()
time.sleep(1)
continue
self.logger.debug("user DN: %s" % userDN)
try:
defaultDelegation = {'logger': self.logger,
'credServerPath': self.config.credentialDir,
'myProxySvr': 'myproxy.cern.ch',
'min_time_left': getattr(self.config, 'minTimeLeft', 36000),
'serverDN': self.config.serverDN,
'uisource': uiSetupScript,
'cleanEnvironment': getattr(self.config, 'cleanEnvironment', False)}
if hasattr(self.config, "cache_area"):
cache_area = self.config.cache_area
defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(cache_area)[0]
except IndexError:
self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % self.config.cache_area)
transfers.task_done()
time.sleep(1)
continue
if getattr(self.config, 'serviceCert', None):
defaultDelegation['server_cert'] = self.config.serviceCert
if getattr(self.config, 'serviceKey', None):
defaultDelegation['server_key'] = self.config.serviceKey
try:
defaultDelegation['userDN'] = userDN
defaultDelegation['group'] = group if group else ''
defaultDelegation['role'] = role if group else ''
self.logger.debug('delegation: %s' % defaultDelegation)
valid_proxy, user_proxy = getProxy(defaultDelegation, self.logger)
except Exception as ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
transfers.task_done()
time.sleep(1)
continue
# TODO: take server from db, right now, take only the first of the list and assuming it valid for all
try:
# TODO: debug u added during info upload. To be fixed soon! For now worked around
fts_server = transfer_list[0]['fts_instance'].split('u')[1]
self.logger.info("Delegating proxy to %s" % fts_server)
context = fts3.Context(fts_server, user_proxy, user_proxy, verify=True)
self.logger.debug(fts3.delegate(context, lifetime=timedelta(hours=48), force=False))
self.logger.info("Proxy delegated. Grouping files by jobId")
jobs = {}
for fileToKill in transfer_list:
# TODO: debug u added during info upload. To be fixed soon! For now worked around
jid = str(fileToKill['fts_id']).split('u')[1]
if jid not in jobs:
jobs[jid] = []
jobs[jid].append(fileToKill)
self.logger.info("Found %s jobIds", len(jobs.keys()))
self.logger.debug("jobIds: %s", jobs.keys)
# list for files killed or failed to
killed = []
too_late = []
for ftsJobId, files in jobs.iteritems():
self.logger.info("Cancelling tranfers in %s" % ftsJobId)
ref_lfns = [str(x['destination_lfn'].split('/store/')[1]) for x in files]
source_lfns = [x['source_lfn'] for x in files]
job_list = fts3.get_job_status(context, ftsJobId, list_files=True)
tx = job_list['files']
# TODO: this workaround is needed to get FTS file id, we may want to add a column in the db?
idListToKill = [x['file_id'] for x in tx
if x['dest_surl'].split('/cms/store/')[1] in ref_lfns]
# needed for the state update
lfnListToKill = [ref_lfns.index(str(x['dest_surl'].split('/cms/store/')[1])) for x in tx
if x['dest_surl'].split('/cms/store/')[1] in ref_lfns]
self.logger.debug("List of ids to cancel for job %s: %s" % (ftsJobId, idListToKill))
res = fts3.cancel(context, ftsJobId, idListToKill)
self.logger.debug('Kill command result: %s' % json.dumps(res))
if not isinstance(res, list):
res = [res]
# Verify if the kill command succeeded
for k, kill_res in enumerate(res):
indexToUpdate = lfnListToKill[k]
if kill_res in ("FINISHEDDIRTY", "FINISHED", "FAILED"):
self.logger.debug(source_lfns[indexToUpdate])
too_late.append(getHashLfn(source_lfns[indexToUpdate]))
else:
killed.append(getHashLfn(source_lfns[indexToUpdate]))
# TODO: decide how to update status for too_late files
killed += too_late
self.logger.debug('Updating status of killed files: %s' % killed)
if len(killed) > 0:
data = dict()
data['asoworker'] = self.config.asoworker
data['subresource'] = 'updateTransfers'
data['list_of_ids'] = killed
data['list_of_transfer_state'] = ["KILLED" for _ in killed]
self.oracleDB.post(self.config.oracleFileTrans,
data=encodeRequest(data))
self.logger.debug("Marked killed %s" % killed)
except:
# TODO: split and improve try/except
self.logger.exception('Kill command failed')
transfers.task_done()
def __init__(self, user, tfc_map, config):
"""
store the user and tfc the worker
"""
self.user = user[0]
self.group = user[1]
self.role = user[2]
self.tfc_map = tfc_map
self.config = config
self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir
logging.basicConfig(level=config.log_level)
self.logger = logging.getLogger('AsyncTransfer-Worker-%s' % self.user)
formatter = getCommonLogFormatter(self.config)
for handler in logging.getLogger().handlers:
handler.setFormatter(formatter)
self.pfn_to_lfn_mapping = {}
self.max_retry = config.max_retry
self.uiSetupScript = getattr(self.config, 'UISetupScript', None)
self.submission_command = getattr(self.config, 'submission_command', 'glite-transfer-submit')
self.cleanEnvironment = ''
self.userDN = ''
self.init = True
if getattr(self.config, 'cleanEnvironment', False):
self.cleanEnvironment = 'unset LD_LIBRARY_PATH; unset X509_USER_CERT; unset X509_USER_KEY;'
self.logger.debug("Trying to get DN for %s" % self.user)
try:
self.userDN = getDNFromUserName(self.user, self.logger, ckey=self.config.opsProxy, cert=self.config.opsProxy)
except Exception as ex:
msg = "Error retrieving the user DN"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
self.init = False
return
if not self.userDN:
self.init = False
return
defaultDelegation = {'logger': self.logger,
'credServerPath' : self.config.credentialDir,
# It will be moved to be getfrom couchDB
'myProxySvr': 'myproxy.cern.ch',
'min_time_left' : getattr(self.config, 'minTimeLeft', 36000),
'serverDN' : self.config.serverDN,
'uisource' : self.uiSetupScript,
'cleanEnvironment' : getattr(self.config, 'cleanEnvironment', False)}
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace=self.config.pluginDir)
self.commandTimeout = 1200
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(self.config.config_database)
self.fts_server_for_transfer = getFTServer("T1_UK_RAL", 'getRunningFTSserver', self.config_db, self.logger)
self.cache_area=""
if hasattr(self.config, "cache_area"):
self.cache_area = self.config.cache_area
query = {'key':self.user}
try:
self.user_cache_area = self.db.loadView('DBSPublisher', 'cache_area', query)['rows']
self.cache_area = "https://"+self.user_cache_area[0]['value'][0]+self.user_cache_area[0]['value'][1]+"/filemetadata"
except Exception as ex:
msg = "Error getting user cache_area."
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
pass
try:
defaultDelegation['myproxyAccount'] = re.compile('https?://([^/]*)/.*').findall(self.cache_area)[0]
except IndexError:
self.logger.error('MyproxyAccount parameter cannot be retrieved from %s . ' % (self.config.cache_area))
if getattr(self.config, 'serviceCert', None):
defaultDelegation['server_cert'] = self.config.serviceCert
if getattr(self.config, 'serviceKey', None):
defaultDelegation['server_key'] = self.config.serviceKey
self.valid_proxy = False
self.user_proxy = None
try:
defaultDelegation['userDN'] = self.userDN
defaultDelegation['group'] = self.group
defaultDelegation['role'] = self.role
self.valid_proxy, self.user_proxy = getProxy(defaultDelegation, self.logger)
except Exception as ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
'A problem occured when contacting couchDB to update last cleanning time: %s'
% e)
return
self.logger.info('LFNs to remove: %s' % len(all_LFNs))
self.logger.debug('LFNs to remove: %s' % all_LFNs)
lfn_to_proxy = {}
for lfnDetails in all_LFNs:
lfn = lfnDetails['value']['lfn']
user = lfn.split('/')[4].split('.')[0]
if not lfn_to_proxy.has_key(user):
valid_proxy = False
try:
userDN = getDNFromUserName(user,
self.logger,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
valid_proxy, userProxy = getProxy(
userDN, "", "", self.defaultDelegation,
self.logger)
except Exception, ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
if valid_proxy:
lfn_to_proxy[user] = userProxy
else:
lfn_to_proxy[user] = self.opsProxy
userProxy = lfn_to_proxy[user]
location = lfnDetails['value']['location']
self.config_db.makeRequest(uri = updateUri, type = "PUT", decode = False)
except Exception, e:
self.logger.exception('A problem occured when contacting couchDB to update last cleanning time: %s' % e)
return
self.logger.info('LFNs to remove: %s' % len(all_LFNs))
self.logger.debug('LFNs to remove: %s' % all_LFNs)
lfn_to_proxy = {}
for lfnDetails in all_LFNs:
lfn = lfnDetails['value']['lfn']
user = lfn.split('/')[4].split('.')[0]
if not lfn_to_proxy.has_key(user):
valid_proxy = False
try:
userDN = getDNFromUserName(user, self.logger, ckey = self.config.opsProxy, cert = self.config.opsProxy)
valid_proxy, userProxy = getProxy(userDN, "", "", self.defaultDelegation, self.logger)
except Exception, ex:
msg = "Error getting the user proxy"
msg += str(ex)
msg += str(traceback.format_exc())
self.logger.error(msg)
if valid_proxy:
lfn_to_proxy[user] = userProxy
else:
lfn_to_proxy[user] = self.opsProxy
userProxy = lfn_to_proxy[user]
location = lfnDetails['value']['location']
self.logger.info("Removing %s from %s" %(lfn, location))
pfn = self.apply_tfc_to_lfn( '%s:%s' %(location, lfn))
if pfn:
