def preInitialization(self): pollInterval = self.config.PhEDExInjector.pollInterval subInterval = self.config.PhEDExInjector.subscribeInterval logging.info("Setting poll interval to %s seconds for inject", pollInterval) # retrieving the node mappings is fickle and can fail quite often # hence only do it once (with retries) and pass it to the workers phedex = PhEDEx({"endpoint": self.config.PhEDExInjector.phedexurl}, "json") try: nodeMappings = phedex.getNodeMap() except Exception: time.sleep(2) try: nodeMappings = phedex.getNodeMap() except Exception: time.sleep(4) nodeMappings = phedex.getNodeMap() myThread = threading.currentThread() myThread.workerThreadManager.addWorker( PhEDExInjectorPoller(self.config, phedex, nodeMappings), pollInterval) if getattr(self.config.PhEDExInjector, "subscribeDatasets", False): # wait a bit for first poll cycle of PhEDExInjectorPoller to complete # hopefully avoids intermingled logs (which can be confusing) time.sleep(2) logging.info("Setting poll interval to %s seconds for subscribe", subInterval) myThread.workerThreadManager.addWorker( PhEDExInjectorSubscriber(self.config, phedex, nodeMappings), subInterval)
def preInitialization(self): pollInterval = self.config.PhEDExInjector.pollInterval subInterval = self.config.PhEDExInjector.subscribeInterval logging.info("Setting poll interval to %s seconds for inject" % pollInterval) myThread = threading.currentThread() myThread.workerThreadManager.addWorker( PhEDExInjectorPoller(self.config), pollInterval) if getattr(self.config.PhEDExInjector, "subscribeDatasets", False): logging.info("Setting poll interval to %s seconds for subscribe" % subInterval) myThread.workerThreadManager.addWorker( PhEDExInjectorSubscriber(self.config), subInterval)
def testNormalModeSubscriptions(self): """ _testNormalModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on normal operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig() phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json") try: nodeMappings = phedex.getNodeMap() except Exception: time.sleep(2) try: nodeMappings = phedex.getNodeMap() except Exception: time.sleep(4) nodeMappings = phedex.getNodeMap() subscriber = PhEDExInjectorSubscriber(config, phedex, nodeMappings) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual( subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual( subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual( subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual( subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual( subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual( subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual( subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual( subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual( subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData( "SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1" )[0].fetchall() self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed") result = myThread.dbi.processData( "SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0" )[0].fetchall() self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed") # Reset and run again and make sure that no duplicate subscriptions are created myThread.dbi.processData( "UPDATE dbsbuffer_dataset_subscription SET subscribed = 0") subscriber.algorithm({}) self.assertEqual(len(subscriptions[self.testDatasetA]), 3) self.assertEqual(len(subscriptions[self.testDatasetB]), 2) return
def testSafeModeSubscriptions(self): """ _testSafeModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on safe operation mode, make sure that the flow of subscriptions obeys the rule laid in the subscriber documentation. Check that the requests are correct. """ config = self.createConfig(safeMode = True) self.stuffDatabase() topSubscription, mergeSubscription = self.stuffWMBS() # Start the subscriber subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) # Run once, this means that all custodial and non-custodial subscriptions # will be made but none will be Move subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as partially subscribed") # Now finish the Processing subscription and run the algorithm again topSubscription.markFinished() subscriber.algorithm({}) self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed again to custodial site") moveCount = 0 for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) if subInfo["move"] == "y": moveCount += 1 else: self.fail("Dataset A was subscribed to a wrong site %s" % site) self.assertEqual(moveCount, 1, "Move subscription was not made") self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was susbcribed again") result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 2")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed") return
def testUnsafeModeSubscriptions(self): """ _testUnsafeModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on unsafe operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig(safeMode = False) subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed") return
def testTier0ModeSubscriptions(self): """ _testTier0ModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on tier0 operation mode, custodial moves are made on block level. """ self.stuffDatabase(tier0Mode = True) self.stuffWMBS() config = self.createConfig(tier0Mode = True) subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T0_CH_CERN and T1_US_FNAL # Non-custodial at T2_US_Vanderbilt # Autoapproved at CERN # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] if subInfo['level'] == 'block': self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["node"], "T0_CH_CERN_MSS", "Wrong node for dataset A, block subscription.") self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") elif site == "T0_CH_CERN_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") elif site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") else: self.fail("Dataset A was subscribed to a wrong site %s" % site) self.assertEqual(len([x for x in subInfoA if x['level'] == 'block']), 1) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, custodial to T0_CH_CERN_MSS and T1_UK_RAL_MSS # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions) subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] if site == "T0_CH_CERN_MSS" or site == "T1_UK_RAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 5, "Custodial move datasets were marked as subscribed") # Delete the wmbs entries myThread.dbi.processData("DELETE FROM wmbs_workflow") subscriber.algorithm({}) self.assertEqual(len(subscriptions[self.testDatasetA]), 5) self.assertTrue(self.testDatasetB in subscriptions) self.assertEqual(len(subscriptions[self.testDatasetB]), 3) self.assertEqual(len([x for x in subscriptions[self.testDatasetB] if x['level'] == 'block']), 1) return