def setUp(self): # stolen from CMSSWExecutor_t. thanks, dave self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile')) self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("DeleterTask") stepHelper = step = self.task.makeStep("DeleteTest") self.step = stepHelper.data self.actualStep = stepHelper template = DeleteTemplate() template(self.step) self.helper = template.helper(self.step) self.executor = StepFactory.getStepExecutor(self.actualStep.stepType()) taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.insert(0, self.testDir) sys.path.insert(0, self.sandboxDir) self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/test" self.dbsURL = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="srm-cms.cern.ch") locationAction.execute(siteName="se.fnal.gov") self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/%s/PromptReco-v1/RECO" % makeUUID() self.testDatasetB = "/%s/CRUZET11-v1/RAW" % makeUUID() self.phedex = PhEDEx({"endpoint": self.phedexURL}, "json") return
def setUp(self): """ Mimic remains of destroyed job """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.nJobs = 10 os.mkdir('test') os.mkdir('test/Basic') os.mkdir('test/Basic/Crap') os.mkdir('test/Basic/Crap/JobCollection_1') for i in range(self.nJobs): os.mkdir('test/Basic/Crap/JobCollection_1/Job_%i' % (i)) self.logLevel = 'INFO' #There must be a submit directory for jobSubmitter if not os.path.isdir('submit'): os.mkdir('submit') #There must be a log directory for jobArchiver if not os.path.isdir('logs'): os.mkdir('logs')
def setUp(self): """ _setUp_ setUp function for unittest """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) myThread = threading.currentThread() self.bufferFactory = DAOFactory( package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.bufferFactory( classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov") locationAction.execute(siteName="malpaquet") return
def setUp(self): "make a logger instance and create tables" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema()
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.nSites = 2 locationAction = daofactory(classname="Locations.New") for site in range(self.nSites): locationAction.execute(siteName="site%i" % site, seName="site%i.cern.ch" % site) return
def setUp(self): """ _setUp_ setUp function for unittest """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"], useDefault = False) self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False) self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) self.buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation") locationAction.execute(siteName = "se1.cern.ch") locationAction.execute(siteName = "se1.fnal.gov") locationAction.execute(siteName = "malpaquet") self.dbsUrl = "https://localhost:1443/dbs/dev/global/DBSWriter" self.dbsApi = None return
def setUp(self): myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.tearDown() self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database" ], useDefault=False) self.daoFactory = DAOFactory(package="WMCore.BossAir", logger=myThread.logger, dbinterface=myThread.dbi) resourceControl = ResourceControl() resourceControl.insertSite(siteName='Xanadu', pnn='se.Xanadu', ceName='Xanadu', plugin="TestPlugin") resourceControl.insertThreshold(siteName = 'Xanadu', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) # Create user wmbsFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) newuser = wmbsFactory(classname="Users.New") newuser.execute(dn="mnorman", group_name="phgroup", role_name="cmsrole")
def __init__(self): """ __init__ Init the DB """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.bufferFactory( classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov") locationAction.execute(siteName="malpaquet") config = self.getConfig() self.dbsUploader = DBSUploadPoller(config=config) return
def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def setUp(self): "make a logger instance and create tables" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() myThread = threading.currentThread() if myThread.dialect == 'MySQL': myThread.create = """ create table test (bind1 varchar(20), bind2 varchar(20)) ENGINE=InnoDB """ myThread.insert = """ insert into test (bind1, bind2) values (:bind1, :bind2) """ myThread.insert_binds = \ [{'bind1': 'value1a', 'bind2': 'value2a'}, \ {'bind1': 'value1b', 'bind2': 'value2b'}, \ {'bind1': 'value1c', 'bind2': 'value2d'}] myThread.select = "select * from test" myThread = threading.currentThread() myThread.transaction = Transaction(myThread.dbi) myThread.transaction.processData(myThread.create) myThread.transaction.processData(myThread.insert, myThread.insert_binds) myThread.transaction.commit() return
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also add some dummy locations. """ self.queueDB = 'workqueue_t' self.queueInboxDB = 'workqueue_t_inbox' self.globalQDB = 'workqueue_t_global' self.globalQInboxDB = 'workqueue_t_global_inbox' self.localQDB = 'workqueue_t_local' self.localQInboxDB = 'workqueue_t_local_inbox' self.localQDB2 = 'workqueue_t_local2' self.localQInboxDB2 = 'workqueue_t_local2_inbox' self.setSchema() self.testInit = TestInit('WorkQueueTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) self.testInit.setupCouch(self.queueDB, *self.couchApps) self.testInit.setupCouch(self.queueInboxDB, *self.couchApps) self.testInit.setupCouch(self.globalQDB, *self.couchApps) self.testInit.setupCouch(self.globalQInboxDB , *self.couchApps) self.testInit.setupCouch(self.localQDB, *self.couchApps) self.testInit.setupCouch(self.localQInboxDB, *self.couchApps) self.testInit.setupCouch(self.localQDB2, *self.couchApps) self.testInit.setupCouch(self.localQInboxDB2, *self.couchApps) self.workDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://bogus.cern.ch/bogus" self.dbsURL = "https://bogus.cern.ch/bogus" EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True) self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault = False) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO" self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW" return
def setUp(self): self.testInit = TestInit(__file__) self.testInit.setLogging(logLevel=logging.DEBUG) self.testDir = self.testInit.generateWorkDir() self.config = getConfig(self.testDir) # mock generator instance to communicate some configuration values self.generator = utils.AlertGeneratorMock(self.config)
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) # We need to set sites in the locations table daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="Locations.New") locationAction.execute(siteName="site1", pnn="T2_CH_CERN") locationAction.execute(siteName="site2", pnn="malpaquet") locationAction.execute(siteName="site3", pnn="badse.cern.ch")
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="Locations.New") locationAction.execute(siteName="T1_US_FNAL", pnn="T1_US_FNAL_Disk") locationAction.execute(siteName="T2_CH_CERN", pnn="T2_CH_CERN") self.testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") self.testWorkflow.create() self.performanceParams = {'timePerEvent': 12, 'memoryRequirement': 2300, 'sizePerEvent': 400} return
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the DBSBuffer tables. Also add some dummy locations. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) self.daoFactory2 = DAOFactory(package="WMComponent.DBSUpload.Database", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.daoFactory( classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov")
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also, create some dummy locations. This doesn't start server automatically. You need to start server before - make sure change self.server_url, if it is not the same as given one - localhost:8080. WMCORE/src/python/WMCore/WebTools/Root.py --ini=WMCORE/src/python/WMCore/HTTPFrontEnd/WMBSDefaultConfig.py """ self.server_url = 'http://localhost:8081' self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.daofactory(classname="Locations.New") locationAction.execute(siteName="test.site.ch") locationAction.execute(siteName="base.site.ch") testSubscription, testFileA, testFileB, testFileC = \ self.createSubscriptionWithFileABC() self.createTestJob(testSubscription, 'TestJob1', testFileA) self.createTestJob(testSubscription, 'TestJob2', testFileB) self.createTestJob(testSubscription, 'TestJob3', testFileC) return
def setUp(self): """ _setUp_ Install schema and create a DAO factory for WMBS. """ EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=False) self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMCore.ResourceControl", "WMCore.BossAir" ], useDefault=False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.baDaoFactory = DAOFactory(package="WMCore.BossAir", logger=myThread.logger, dbinterface=myThread.dbi) self.insertRunJob = self.baDaoFactory(classname="NewJobs") self.insertState = self.baDaoFactory(classname="NewState") states = ['PEND', 'RUN', 'Idle', 'Running'] self.insertState.execute(states) self.tempDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="Locations.New") locationAction.execute(siteName='s1', seName="somese.cern.ch") locationAction.execute(siteName='s2', seName="otherse.cern.ch") self.testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") self.testWorkflow.create() return
def setUp(self): self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMQuality.TestDB"], useDefault=False) return
def setUp(self): self.testInit = TestInit(__file__) self.testInit.setLogging() self.testDir = self.testInit.generateWorkDir() sys.path.insert( 0, os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/WMRuntime_t/Scripts_t"))
def setUp(self): self.testInit = TestInit(__file__) self.testInit.setLogging(logLevel=logging.DEBUG) self.addr = "tcp://127.0.0.1:5557" self.ctrl = "tcp://127.0.0.1:5559" # simple printer Alert messages handler self.printer = lambda x: sys.stdout.write("printer handler: '%s'\n" % str(x))
def setUp(self): """ setup for test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema()
def setUp(self): """set up""" self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir if PY3: self.assertItemsEqual = self.assertCountEqual
def setUp(self): "make a logger instance and create tables" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMQuality.TestDB"], useDefault=False) self.selectSQL = "SELECT * FROM test_tableb"
def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL") myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL") myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN") self.fileset1 = Fileset(name="TestFileset1") for fileNum in range(11): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset=self.fileset1, workflow=workflow1, split_algo="Harvest", type="Harvesting") self.subscription1.create() self.configFile = EmulatorSetup.setupWMAgentConfig() return
def setUp(self): """ _setUp_ Setup logging. """ self.testInit = TestInit(__file__) self.testInit.setLogging() return
def setUp(self): """ _setUp_ Create a temporary file to presist the JobPackage to. """ self.testInit = TestInit(__file__) self.persistFile = os.path.join(self.testInit.generateWorkDir(), "JobPackage.pkl") return
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the DBSBuffer tables. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema( customModules=["WMComponent.DBSBuffer.Database"], useDefault=False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) datasetAction = self.daoFactory(classname="NewDataset") datasetListAction = self.daoFactory(classname="ListDataset") datasetAction.execute( datasetPath="/Cosmics/CRUZET09-PromptReco-v1/RECO", validStatus="PRODUCTION") listResult = datasetListAction.execute( datasetPath="/Cosmics/CRUZET09-PromptReco-v1/RECO") self.datasetID = listResult[0]["id"] algoAction = self.daoFactory(classname="NewAlgo") algoListAction = self.daoFactory(classname="ListAlgo") algoAction.execute(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="FEVT", psetHash="GIBBERISH", configContent="MOREGIBBERISH") algoAction.execute(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="FEVT", psetHash="GIBBERISH2", configContent="MOREGIBBERISH2") listResult = algoListAction.execute(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="FEVT", psetHash="GIBBERISH", configContent="MOREGIBBERISH") self.algo1ID = listResult[0]["id"] listResult = algoListAction.execute(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="FEVT", psetHash="GIBBERISH2", configContent="MOREGIBBERISH2") self.algo2ID = listResult[0]["id"]