def __init__(self, msConfig, **kwargs):
"""
Provides setup for MSTransferor and MSMonitor classes
:param config: MS service configuration
:param kwargs: can be used to skip the initialization of specific services, such as:
logger: logger object
skipReqMgr: boolean to skip ReqMgr initialization
skipReqMgrAux: boolean to skip ReqMgrAux initialization
skipRucio: boolean to skip Rucio initialization
"""
self.logger = getMSLogger(getattr(msConfig, 'verbose', False), kwargs.get("logger"))
self.msConfig = msConfig
self.logger.info("Configuration including default values:\n%s", self.msConfig)
if not kwargs.get("skipReqMgr", False):
self.reqmgr2 = ReqMgr(self.msConfig['reqmgr2Url'], logger=self.logger)
if not kwargs.get("skipReqMgrAux", False):
self.reqmgrAux = ReqMgrAux(self.msConfig['reqmgr2Url'],
httpDict={'cacheduration': 1.0}, logger=self.logger)
self.phedex = None
self.rucio = None
if not kwargs.get("skipRucio", False):
self.rucio = Rucio(acct=self.msConfig['rucioAccount'],
hostUrl=self.msConfig['rucioUrl'],
authUrl=self.msConfig['rucioAuthUrl'],
configDict={"logger": self.logger, "user_agent": "wmcore-microservices"})
def __init__(self, nworkers=10, name='TaskManager', logger=None):
self.logger = getMSLogger(verbose=True, logger=logger)
self.name = name
self.pids = set()
self.uids = UidSet()
self.tasks = Queue()
self.workers = [Worker(name, self.tasks, self.pids, self.uids, logger) \
for _ in range(0, nworkers)]
def __init__(self, name, taskq, pidq, uidq, logger=None):
self.logger = getMSLogger(verbose=True, logger=logger)
threading.Thread.__init__(self, name=name)
self.exit = 0
self.tasks = taskq
self.pids = pidq
self.uids = uidq
self.daemon = True
self.start()
def __init__(self, reqName, reqData, logger=None, verbose=False):
self.reqName = reqName
self.data = reqData
self.logger = getMSLogger(verbose, logger)
self.inputDataset = ""
self.parentDataset = ""
self.pileupDatasets = set()
self.pileupRSEList = set()
self.campaigns = set()
self.dataCampaignMap = []
# these blocks structure will be key'ed by the block name and value'd by the block size
self.primaryBlocks = {}
self.parentBlocks = {}
# sort of duplicate info, but we need to have a way to link input to parent block(s)
self.childToParentBlocks = {}
# pileup don't need to get resolved into blocks, store only their total size and location
self.secondarySummaries = {}
self.setDataCampaignMap()
self.setInputData()
def __init__(self, dataAcct, quotaFraction, **kwargs):
"""
Executes a basic setup, including proper logging.
:param dataAcct: string with the Rucio account
:param quotaFraction: float point number representing the fraction of the quota
:param kwargs: the supported keyword arguments are:
minimumThreshold: integer value defining the minimum available space required
verbose: logger verbosity
logger: logger object
"""
self.dataAcct = dataAcct
self.quotaFraction = quotaFraction
self.minimumSpace = kwargs["minimumThreshold"]
self.logger = getMSLogger(kwargs.get("verbose"), kwargs.get("logger"))
msg = "RSEQuotas started with parameters: dataAcct=%s, quotaFraction=%s, "
msg += "minimumThreshold=%s GB"
self.logger.info(msg, dataAcct, quotaFraction,
gigaBytes(self.minimumSpace))
self.nodeUsage = {}
self.availableRSEs = set()
self.outOfSpaceNodes = set()
def __init__(self, mode=None, logger=None):
self.logger = getMSLogger(verbose=True, logger=logger)
if mode == 'test':
self.siteInfo = {}
else:
self.siteInfo = self.fetch()
def __init__(self, uConfig, mode=None, logger=None):
self.logger = getMSLogger(verbose=True, logger=logger)
self.siteCache = SiteCache(mode, logger)
self.config = uConfig
self.sites_ready_in_agent = self.siteCache.get('ready_in_agent', [])
self.sites_ready = []
self.sites_not_ready = []
self.all_sites = []
self.sites_banned = self.config.get('sites_banned', [])
# try:
# sites_full = json.loads(open('sites_full.json').read())
# except:
# pass
data = self.siteCache.get('ssb_237', {'csvdata': []})
for siteInfo in data['csvdata']:
if not siteInfo['Tier'] in [0, 1, 2, 3]:
continue
self.all_sites.append(siteInfo['VOName'])
if siteInfo['VOName'] in self.sites_banned:
continue
if self.sites_ready_in_agent and siteInfo[
'VOName'] in self.sites_ready_in_agent:
self.sites_ready.append(siteInfo['VOName'])
elif self.sites_ready_in_agent and not siteInfo[
'VOName'] in self.sites_ready_in_agent:
self.sites_not_ready.append(siteInfo['VOName'])
elif siteInfo['Status'] == 'enabled':
self.sites_ready.append(siteInfo['VOName'])
else:
self.sites_not_ready.append(siteInfo['VOName'])
self.sites_auto_approve = self.config.get('sites_auto_approve')
self.sites_eos = [s for s in self.sites_ready \
if s in ['T2_CH_CERN', 'T2_CH_CERN_HLT']]
self.sites_T3s = [s for s in self.sites_ready if s.startswith('T3_')]
self.sites_T2s = [s for s in self.sites_ready if s.startswith('T2_')]
self.sites_T1s = [s for s in self.sites_ready if s.startswith('T1_')]
self.sites_T0s = [s for s in self.sites_ready if s.startswith('T0_')]
self.sites_T3s_all = [s for s in self.all_sites if s.startswith('T3_')]
self.sites_T2s_all = [s for s in self.all_sites if s.startswith('T2_')]
self.sites_T1s_all = [s for s in self.all_sites if s.startswith('T1_')]
self.sites_T0s_all = [s for s in self.all_sites if s.startswith('T0_')]
self.sites_AAA = list(set(self.sites_ready) - set(['T2_CH_CERN_HLT']))
## could this be an SSB metric ?
self.sites_with_goodIO = self.config.get('sites_with_goodIO', [])
#restrict to those that are actually ON
self.sites_with_goodIO = [
s for s in self.sites_with_goodIO if s in self.sites_ready
]
self.sites_veto_transfer = [
] ## do not prevent any transfer by default
## new site lists for better matching
self.sites_with_goodAAA = self.sites_with_goodIO \
+ ['T3_IN_TIFRCloud', 'T3_US_NERSC'] ## like this for now
self.sites_with_goodAAA = [
s for s in self.sites_with_goodAAA if s in self.sites_ready
]
self.storage = defaultdict(int)
self.disk = defaultdict(int)
self.queue = defaultdict(int)
self.free_disk = defaultdict(int)
self.quota = defaultdict(int)
self.locked = defaultdict(int)
self.cpu_pledges = defaultdict(int)
self.addHocStorage = {
'T2_CH_CERN_T0': 'T2_CH_CERN',
'T2_CH_CERN_HLT': 'T2_CH_CERN',
'T2_CH_CERN_AI': 'T2_CH_CERN',
'T3_IN_TIFRCloud': 'T2_IN_TIFR',
#'T3_US_NERSC' : 'T1_US_FNAL_Disk'
}
## list here the site which can accomodate high memory requests
self.sites_memory = {}
self.sites_mcore_ready = []
mcore_mask = self.siteCache.get('mcore')
if mcore_mask:
self.sites_mcore_ready = \
[s for s in mcore_mask['sites_for_mcore'] if s in self.sites_ready]
else:
pass
for sname in self.all_sites:
self.cpu_pledges[sname] = 1 # will get it later from SSB
self.disk[self.ce2SE(sname)] = 0 # will get it later from SSB
tapes = getNodes('MSS')
for mss in tapes:
if mss in self.sites_banned:
continue # not using these tapes for MC familly
self.storage[mss] = 0
## and get SSB sync
self.fetch_ssb_info()
mss_usage = self.siteCache.get('mss_usage')
sites_space_override = self.config.get('sites_space_override', {})
if mss_usage:
use_field = 'Usable'
for mss in self.storage:
if not mss in mss_usage['Tape'][use_field]:
self.storage[mss] = 0
else:
self.storage[mss] = mss_usage['Tape'][use_field][mss]
if mss in sites_space_override:
self.storage[mss] = sites_space_override[mss]
self.fetch_queue_info()
## and detox info
self.fetch_detox_info(\
buffer_level=self.config.get('DDM_buffer_level', None),\
sites_space_override=sites_space_override)
## transform no disks in veto transfer
for dse, free in viewitems(self.disk):
if free <= 0:
if not dse in self.sites_veto_transfer:
self.sites_veto_transfer.append(dse)
## and glidein info
self.fetch_glidein_info()
def __init__(self, config=None, logger=None):
"""
Initialize MSManager class with given configuration,
logger, ReqMgr2/ReqMgrAux/PhEDEx/Rucio objects,
and start transferor and monitoring threads.
:param config: reqmgr2ms service configuration
:param logger:
"""
self.config = config
self.logger = getMSLogger(getattr(config, 'verbose', False), logger)
self._parseConfig(config)
self.logger.info("Configuration including default values:\n%s",
self.msConfig)
self.statusTrans = {}
self.statusMon = {}
self.statusOutput = {}
self.statusRuleCleaner = {}
self.statusUnmerged = {}
# initialize transferor module
if 'transferor' in self.services:
self.msTransferor = MSTransferor(self.msConfig, logger=self.logger)
thname = 'MSTransferor'
self.transfThread = start_new_thread(
thname, daemon, (self.transferor, 'assigned',
self.msConfig['interval'], self.logger))
self.logger.info("### Running %s thread %s", thname,
self.transfThread.running())
# initialize monitoring module
if 'monitor' in self.services:
self.msMonitor = MSMonitor(self.msConfig, logger=self.logger)
thname = 'MSMonitor'
self.monitThread = start_new_thread(
thname, daemon, (self.monitor, 'staging',
self.msConfig['interval'], self.logger))
self.logger.info("+++ Running %s thread %s", thname,
self.monitThread.running())
# initialize output module
if 'output' in self.services:
reqStatus = ['closed-out', 'announced']
# thread safe cache to keep the last X requests processed in MSOutput
requestNamesCached = deque(
maxlen=self.msConfig.get("cacheRequestSize", 10000))
thname = 'MSOutputConsumer'
self.msOutputConsumer = MSOutput(self.msConfig,
mode=thname,
reqCache=requestNamesCached,
logger=self.logger)
# set the consumer to run twice faster than the producer
consumerInterval = self.msConfig['interval'] // 2
self.outputConsumerThread = start_new_thread(
thname, daemon, (self.outputConsumer, reqStatus,
consumerInterval, self.logger))
self.logger.info("=== Running %s thread %s", thname,
self.outputConsumerThread.running())
thname = 'MSOutputProducer'
self.msOutputProducer = MSOutput(self.msConfig,
mode=thname,
reqCache=requestNamesCached,
logger=self.logger)
self.outputProducerThread = start_new_thread(
thname, daemon, (self.outputProducer, reqStatus,
self.msConfig['interval'], self.logger))
self.logger.info("=== Running %s thread %s", thname,
self.outputProducerThread.running())
# initialize rule cleaner module
if 'ruleCleaner' in self.services:
reqStatus = ['announced', 'aborted-completed', 'rejected']
self.msRuleCleaner = MSRuleCleaner(self.msConfig,
logger=self.logger)
thname = 'MSRuleCleaner'
self.ruleCleanerThread = start_new_thread(
thname, daemon, (self.ruleCleaner, reqStatus,
self.msConfig['interval'], self.logger))
self.logger.info("--- Running %s thread %s", thname,
self.ruleCleanerThread.running())
# initialize unmerged module
if 'unmerged' in self.services:
self.msUnmerged = MSUnmerged(self.msConfig, logger=self.logger)
thname = 'MSUnmerged'
self.unmergedThread = start_new_thread(
thname, daemonOpt,
(self.unmerged, self.msConfig['interval'], self.logger))
self.logger.info("--- Running %s thread %s", thname,
self.unmergedThread.running())