def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.enabled = config.RucioInjector.enabled
# dataset rule creation has a larger polling cycle
self.pollRules = config.RucioInjector.pollIntervalRules
self.lastRulesExecTime = 0
self.createBlockRules = config.RucioInjector.createBlockRules
self.containerDiskRuleParams = config.RucioInjector.containerDiskRuleParams
self.containerDiskRuleRSEExpr = config.RucioInjector.containerDiskRuleRSEExpr
self.skipRulesForTiers = config.RucioInjector.skipRulesForTiers
self.listTiersToInject = config.RucioInjector.listTiersToInject
if config.RucioInjector.metaDIDProject not in RUCIO_VALID_PROJECT:
msg = "Component configured with an invalid 'project' DID: %s"
raise RucioInjectorException(msg % config.RucioInjector.metaDIDProject)
self.metaDIDProject = dict(project=config.RucioInjector.metaDIDProject)
# setup cache for container and blocks (containers can be much longer, make 6 days now)
self.containersCache = MemoryCache(config.RucioInjector.cacheExpiration * 3, set())
self.blocksCache = MemoryCache(config.RucioInjector.cacheExpiration, set())
self.scope = getattr(config.RucioInjector, "scope", "cms")
self.rucioAcct = config.RucioInjector.rucioAccount
self.rucio = Rucio(acct=self.rucioAcct,
hostUrl=config.RucioInjector.rucioUrl,
authUrl=config.RucioInjector.rucioAuthUrl,
configDict={'logger': self.logger})
# metadata dictionary information to be added to block/container rules
# cannot be a python dictionary, but a JSON string instead
self.metaData = json.dumps(dict(agentHost=config.Agent.hostName,
userAgent=config.Agent.agentName))
self.testRSEs = config.RucioInjector.RSEPostfix
self.filesToRecover = []
# output data placement has a different behaviour between T0 and Production agents
if hasattr(config, "Tier0Feeder"):
logging.info("RucioInjector running on a T0 WMAgent")
self.isT0agent = True
else:
self.isT0agent = False
if not self.listTiersToInject:
logging.info("Component configured to inject all the data tiers")
else:
logging.info("Component configured to only inject data for data tiers: %s",
self.listTiersToInject)
logging.info("Component configured to skip container rule creation for data tiers: %s",
self.skipRulesForTiers)
logging.info("Component configured to create block rules: %s", self.createBlockRules)
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.enabled = config.RucioInjector.enabled
# dataset rule creation has a larger polling cycle
self.pollRules = config.RucioInjector.pollIntervalRules
self.lastRulesExecTime = 0
self.createBlockRules = config.RucioInjector.createBlockRules
self.skipRulesForTiers = config.RucioInjector.skipRulesForTiers
self.listTiersToInject = config.RucioInjector.listTiersToInject
# setup cache for container and blocks (containers can be much longer, make 6 days now)
self.containersCache = MemoryCache(
config.RucioInjector.cacheExpiration * 3, set())
self.blocksCache = MemoryCache(config.RucioInjector.cacheExpiration,
set())
self.scope = getattr(config.RucioInjector, "scope", "cms")
self.rucioAcct = config.RucioInjector.rucioAccount
self.rucio = Rucio(acct=self.rucioAcct,
hostUrl=config.RucioInjector.rucioUrl,
authUrl=config.RucioInjector.rucioAuthUrl,
configDict={'logger': self.logger})
# metadata dictionary information to be added to block/container rules
# cannot be a python dictionary, but a JSON string instead
self.metaData = json.dumps(
dict(agentHost=config.Agent.hostName,
userAgent=config.Agent.agentName))
self.testRSEs = config.RucioInjector.RSEPostfix
self.filesToRecover = []
logging.info(
"Component configured to only inject data for data tiers: %s",
self.listTiersToInject)
logging.info(
"Component configured to skip container rule creation for data tiers: %s",
self.skipRulesForTiers)
logging.info("Component configured to create block rules: %s",
self.createBlockRules)
def testBasics(self):
cache = MemoryCache(1, [])
self.assertItemsEqual(cache.getCache(), [])
cache.setCache(["item1", "item2"])
self.assertItemsEqual(cache.getCache(), ["item1", "item2"])
# wait for cache to expiry, wait for 2 secs
sleep(2)
self.assertRaises(MemoryCacheException, cache.getCache)
cache.setCache(["item4"])
# and the cache is alive again
self.assertItemsEqual(cache.getCache(), ["item4"])
def testCacheDict(self):
cache = MemoryCache(2, {})
self.assertCountEqual(cache.getCache(), {}) if PY3 else self.assertItemsEqual(cache.getCache(), {})
cache.setCache({"item1": 11, "item2": 22})
self.assertCountEqual(cache.getCache(), {"item1": 11, "item2": 22}) if PY3 else self.assertItemsEqual(cache.getCache(), {"item1": 11, "item2": 22})
cache.addItemToCache({"item3": 33})
self.assertCountEqual(cache.getCache(), {"item1": 11, "item2": 22, "item3": 33}) if PY3 else self.assertItemsEqual(cache.getCache(), {"item1": 11, "item2": 22, "item3": 33})
self.assertTrue("item2" in cache)
self.assertFalse("item222" in cache)
# test exceptions
self.assertRaises(TypeError, cache.addItemToCache, "item4")
self.assertRaises(TypeError, cache.addItemToCache, ["item4"])
def testCacheList(self):
cache = MemoryCache(2, [])
self.assertCountEqual(cache.getCache(), []) if PY3 else self.assertItemsEqual(cache.getCache(), [])
cache.setCache(["item1", "item2"])
self.assertCountEqual(cache.getCache(), ["item1", "item2"]) if PY3 else self.assertItemsEqual(cache.getCache(), ["item1", "item2"])
cache.addItemToCache("item3")
self.assertCountEqual(cache.getCache(), ["item1", "item2", "item3"]) if PY3 else self.assertItemsEqual(cache.getCache(), ["item1", "item2", "item3"])
cache.addItemToCache(["item4"])
self.assertCountEqual(cache.getCache(), ["item1", "item2", "item3", "item4"]) if PY3 else self.assertItemsEqual(cache.getCache(), ["item1", "item2", "item3", "item4"])
cache.addItemToCache(set(["item5"]))
self.assertCountEqual(cache.getCache(), ["item1", "item2", "item3", "item4", "item5"]) if PY3 else self.assertItemsEqual(cache.getCache(), ["item1", "item2", "item3", "item4", "item5"])
self.assertTrue("item2" in cache)
self.assertFalse("item222" in cache)
def testCacheSet(self):
cache = MemoryCache(2, set())
self.assertItemsEqual(cache.getCache(), set())
cache.setCache(set(["item1", "item2"]))
self.assertItemsEqual(cache.getCache(), ["item1", "item2"])
cache.addItemToCache("item3")
self.assertItemsEqual(cache.getCache(), ["item1", "item2", "item3"])
cache.addItemToCache(["item4"])
self.assertItemsEqual(cache.getCache(),
["item1", "item2", "item3", "item4"])
cache.addItemToCache(set(["item5"]))
self.assertItemsEqual(cache.getCache(),
["item1", "item2", "item3", "item4", "item5"])
self.assertTrue("item2" in cache)
self.assertFalse("item222" in cache)
def testSetDiffTypes(self):
cache = MemoryCache(2, set())
self.assertItemsEqual(cache.getCache(), set())
cache.setCache({"item1", "item2"})
self.assertRaises(TypeError, cache.setCache, ["item3"])