def testAddCommit(self): """ _testAddCommit_ Test the add() and commit() methods of the JobGroup class. Verify that jobs are not returned from getJobs() until commit() has been called. """ testJob = Job() testJobGroup = JobGroup() assert len(testJobGroup.getJobs()) == 0, \ "ERROR: JobGroup has jobs before jobs have been added." testJobGroup.add(testJob) assert len(testJobGroup.getJobs()) == 0, \ "ERROR: JobGroup has jobs commit() was called." testJobGroup.commit() assert len(testJobGroup.getJobs()) == 1, \ "ERROR: JobGroup has wrong number of jobs." assert testJob in testJobGroup.getJobs(), \ "ERROR: JobGroup has unknown jobs." return
def loadData(self): """ _loadData_ Load all data that is associated with the jobgroup. This includes loading all the subscription information, the output fileset information and all the jobs that are associated with the group. """ existingTransaction = self.beginTransaction() if self.id < 0 or self.uid == None: self.load() self.subscription.loadData() self.output.loadData() loadAction = self.daofactory(classname = "JobGroup.LoadJobs") result = loadAction.execute(self.id, conn = self.getDBConn(), transaction = self.existingTransaction()) self.jobs = [] self.newjobs = [] for jobID in result: newJob = Job(id = jobID["id"]) newJob.loadData() self.add(newJob) WMJobGroup.commit(self) self.commitTransaction(existingTransaction) return
def testLen(self): """ __testLen__ Test that the __len__ function will actualy return the correct length. """ #This is simple. It should just have a length equal to the number of committed #And yet to be committed jobs testJobA = Job() testJobB = Job() testJobC = Job() testJobD = Job() testJobGroup = JobGroup(jobs = [testJobA, testJobB]) testJobGroup.commit() self.assertEqual(len(testJobGroup), 2) testJobGroup.add(testJobC) self.assertEqual(len(testJobGroup), 3) testJobGroup.commit() testJobGroup.add(testJobD) self.assertEqual(len(testJobGroup), 4) return
def loadData(self): """ _loadData_ Load all data that is associated with the jobgroup. This includes loading all the subscription information, the output fileset information and all the jobs that are associated with the group. """ existingTransaction = self.beginTransaction() if self.id < 0 or self.uid == None: self.load() self.subscription.loadData() self.output.loadData() loadAction = self.daofactory(classname="JobGroup.LoadJobs") result = loadAction.execute(self.id, conn=self.getDBConn(), transaction=self.existingTransaction()) self.jobs = [] self.newjobs = [] for jobID in result: newJob = Job(id=jobID["id"]) newJob.loadData() self.add(newJob) WMJobGroup.commit(self) self.commitTransaction(existingTransaction) return
def testLen(self): """ __testLen__ Test that the __len__ function will actualy return the correct length. """ # This is simple. It should just have a length equal to the number of committed # And yet to be committed jobs testJobA = Job() testJobB = Job() testJobC = Job() testJobD = Job() testJobGroup = JobGroup(jobs=[testJobA, testJobB]) testJobGroup.commit() self.assertEqual(len(testJobGroup), 2) testJobGroup.add(testJobC) self.assertEqual(len(testJobGroup), 3) testJobGroup.commit() testJobGroup.add(testJobD) self.assertEqual(len(testJobGroup), 4) return
def commitBulk(self): """ Creates jobs in a group instead of singly, as is done in jobGroup.commit() """ myThread = threading.currentThread() if self.id == -1: myThread.transaction.begin() #existingTransaction = self.beginTransaction() self.create() #self.commitTransaction(existingTransaction) myThread.transaction.commit() existingTransaction = self.beginTransaction() listOfJobs = [] for job in self.newjobs: #First do all the header stuff if job["id"] != None: continue job["jobgroup"] = self.id if job["name"] == None: job["name"] = makeUUID() listOfJobs.append(job) bulkAction = self.daofactory(classname = "Jobs.New") result = bulkAction.execute(jobList = listOfJobs) #Use the results of the bulk commit to get the jobIDs fileDict = {} for job in listOfJobs: job['id'] = result[job['name']] fileDict[job['id']] = [] for file in job['input_files']: fileDict[job['id']].append(file['id']) maskAction = self.daofactory(classname = "Masks.New") maskAction.execute(jobList = listOfJobs, conn = self.getDBConn(), \ transaction = self.existingTransaction()) fileAction = self.daofactory(classname = "Jobs.AddFiles") fileAction.execute(jobDict = fileDict, conn = self.getDBConn(), \ transaction = self.existingTransaction()) WMJobGroup.commit(self) self.commitTransaction(existingTransaction) return
def __init__(self, subscription = None, jobs = None, id = -1, uid = None, location = None): WMBSBase.__init__(self) WMJobGroup.__init__(self, subscription=subscription, jobs = jobs) self.id = id self.lastUpdate = None self.uid = uid if location != None: self.setSite(location) return
def commitBulk(self): """ Creates jobs in a group instead of singly, as is done in jobGroup.commit() """ myThread = threading.currentThread() if self.id == -1: myThread.transaction.begin() #existingTransaction = self.beginTransaction() self.create() #self.commitTransaction(existingTransaction) myThread.transaction.commit() existingTransaction = self.beginTransaction() listOfJobs = [] for job in self.newjobs: #First do all the header stuff if job["id"] != None: continue job["jobgroup"] = self.id if job["name"] == None: job["name"] = makeUUID() listOfJobs.append(job) bulkAction = self.daofactory(classname="Jobs.New") result = bulkAction.execute(jobList=listOfJobs) #Use the results of the bulk commit to get the jobIDs fileDict = {} for job in listOfJobs: job['id'] = result[job['name']] fileDict[job['id']] = [] for file in job['input_files']: fileDict[job['id']].append(file['id']) maskAction = self.daofactory(classname="Masks.New") maskAction.execute(jobList = listOfJobs, conn = self.getDBConn(), \ transaction = self.existingTransaction()) fileAction = self.daofactory(classname="Jobs.AddFiles") fileAction.execute(jobDict = fileDict, conn = self.getDBConn(), \ transaction = self.existingTransaction()) WMJobGroup.commit(self) self.commitTransaction(existingTransaction) return
def execute(self, *args, **kwargs): self.logger.debug("Transforming old specs into jobs.") # mapping to cache job def - blocks association blocks = {} regroupjobs = {} ## grouping in a dictionary can happen here for job in args[0]: if job.jobDefinitionID in regroupjobs: regroupjobs[job.jobDefinitionID].append(job) else: regroupjobs[job.jobDefinitionID] = [job] jobgroups = [] ## here converting the grouping into proper JobGroup-Jobs for jobdef in regroupjobs: jobgroup = blocks.get(jobdef, None) if jobgroup is None: configreq = {'subresource': 'jobgroup', 'subjobdef': jobdef, 'subuser': kwargs['task']['tm_user_dn']} self.logger.debug("Retrieving %d jobdef information from task manager db: %s" %(jobdef, str(configreq))) jobgroup = self.server.get(self.resturl, data = configreq) self.logger.debug("Jobgroup information in task manager: %s" % str(jobgroup)) jobgroup = jobgroup[0]['result'][0] blocks[jobdef] = jobgroup['tm_data_blocks'] jg = WMJobGroup() for job in regroupjobs[jobdef]: parser = PassThroughOptionParser() parser.add_option('--inputFile', dest='inputfiles', type='string') parser.add_option('--runAndLumis', dest='runlumis', type='string') parser.add_option('--availableSites', dest='allsites', type='string') parser.add_option('--jobNumber', dest='jobnum', type='int') (options, args) = parser.parse_args(shlex.split(job.jobParameters)) jj = WMJob() jj['input_files'] = [] for infile in literal_eval(options.inputfiles): jj['input_files'].append({'lfn': infile, 'block': blocks[jobdef], 'locations': [ss for ss in literal_eval(options.allsites)]}) if options.runlumis: jj['mask']['runAndLumis'] = literal_eval(options.runlumis) jj['panda_oldjobid'] = job.PandaID jj['jobnum'] = options.jobnum jg.add(jj) setattr(jg, 'blocks', blocks[jobdef]) jg.commit() jobgroups.append(jg) return Result(task=kwargs['task'], result=jobgroups)
def testCreate(self): """ _testCreate_ Test the JobGroup constructor and passing different job containers into it. """ testSubscription = Subscription() testJobGroupA = JobGroup(subscription = testSubscription) assert testJobGroupA.subscription == testSubscription, \ "ERROR: Failed to pass subscription in constructor" assert len(testJobGroupA.jobs) == 0 and len(testJobGroupA.newjobs) == 0, \ "ERROR: JobGroup not empty on creation" testJobA = Job() testJobB = Job() testJobGroupB = JobGroup(jobs = [testJobA, testJobB]) assert testJobGroupB.jobs == [], \ "ERROR: Jobs committed to jobgroup too soon." jobGroupJobs = testJobGroupB.newjobs goldenJobs = [testJobA, testJobB] for job in jobGroupJobs: assert job in goldenJobs, \ "ERROR: Extra job in job group" goldenJobs.remove(job) assert len(goldenJobs) == 0, \ "ERROR: Job missing from job group" testJobGroupC = JobGroup(jobs = testJobA) assert testJobGroupC.jobs == [], \ "ERROR: Jobs committed to jobgroup too soon." jobGroupJobs = testJobGroupC.newjobs assert len(jobGroupJobs) == 1, \ "ERROR: Wrong number of jobs in jobgroup." assert testJobA in jobGroupJobs, \ "ERROR: Wrong job in jobgroup." return
def __init__(self, subscription=None, jobs=None, id=-1, uid=None, location=None): WMBSBase.__init__(self) WMJobGroup.__init__(self, subscription=subscription, jobs=jobs) self.id = id self.lastUpdate = None self.uid = uid if location != None: self.setSite(location) return
def commit(self): """ _commit_ Write any new jobs to the database, creating them in the database if necessary. """ existingTransaction = self.beginTransaction() if self.id == -1: self.create() for j in self.newjobs: j.create(group = self) WMJobGroup.commit(self) self.commitTransaction(existingTransaction) return
def commit(self): """ _commit_ Write any new jobs to the database, creating them in the database if necessary. """ existingTransaction = self.beginTransaction() if self.id == -1: self.create() for j in self.newjobs: j.create(group=self) WMJobGroup.commit(self) self.commitTransaction(existingTransaction) return
def testGetJobs(self): """ _testGetJobs_ Verify that the getJobs() method of the JobGroup class returns the correct output for each output container type it supports. """ testJobA = Job() testJobB = Job() testJobGroup = JobGroup(jobs=[testJobA, testJobB]) testJobGroup.commit() assert len(testJobGroup.getJobs()) == 2, "ERROR: Wrong number of jobs in job group" goldenJobs = [testJobA, testJobB] for job in testJobGroup.getJobs(): assert job in goldenJobs, "ERROR: Unknown Job in JobGroup." goldenJobs.remove(job) assert len(goldenJobs) == 0, "ERROR: Jobs are missing from the JobGroup." goldenIDs = [] goldenIDs.append(testJobA["id"]) goldenIDs.append(testJobB["id"]) for jobID in testJobGroup.getJobs(type="id"): assert jobID in goldenIDs, "ERROR: Unknown JobID in JobGroup" goldenIDs.remove(jobID) assert len(goldenIDs) == 0, "ERROR: Job IDs are missing from the JobGroup." return
def testAddOutput(self): """ _testAddOutput_ Test the JobGroup's addOutput() method. Verify that files are committed to the output fileset immediately and are available from the getOutput() method. """ testFile = File() testJobGroup = JobGroup() assert len(testJobGroup.getOutput()) == 0, \ "ERROR: Files in the output fileset before anything has been added." testJobGroup.addOutput(testFile) assert len(testJobGroup.getOutput()) == 1, \ "ERROR: Unknown number of files in JobGroup output fileset." assert testFile in testJobGroup.getOutput(), \ "ERROR: Unknown file in the JobGroup output fileset." return
def testAddOutput(self): """ _testAddOutput_ Test the JobGroup's addOutput() method. Verify that files are committed to the output fileset immediately and are available from the getOutput() method. """ testFile = File() testJobGroup = JobGroup() assert len(testJobGroup.getOutput()) == 0, "ERROR: Files in the output fileset before anything has been added." testJobGroup.addOutput(testFile) assert len(testJobGroup.getOutput()) == 1, "ERROR: Unknown number of files in JobGroup output fileset." assert testFile in testJobGroup.getOutput(), "ERROR: Unknown file in the JobGroup output fileset." return
def testGetJobs(self): """ _testGetJobs_ Verify that the getJobs() method of the JobGroup class returns the correct output for each output container type it supports. """ testJobA = Job() testJobB = Job() testJobGroup = JobGroup(jobs = [testJobA, testJobB]) testJobGroup.commit() assert len(testJobGroup.getJobs()) == 2, \ "ERROR: Wrong number of jobs in job group" goldenJobs = [testJobA, testJobB] for job in testJobGroup.getJobs(): assert job in goldenJobs, \ "ERROR: Unknown Job in JobGroup." goldenJobs.remove(job) assert len(goldenJobs) == 0, \ "ERROR: Jobs are missing from the JobGroup." goldenIDs = [] goldenIDs.append(testJobA["id"]) goldenIDs.append(testJobB["id"]) for jobID in testJobGroup.getJobs(type = "id"): assert jobID in goldenIDs, \ "ERROR: Unknown JobID in JobGroup" goldenIDs.remove(jobID) assert len(goldenIDs) == 0, \ "ERROR: Job IDs are missing from the JobGroup." return
def execute(self, *args, **kwargs): self.logger.debug("Transforming old specs into jobs.") # mapping to cache job def - blocks association blocks = {} regroupjobs = {} ## grouping in a dictionary can happen here for job in args[0]: if job.jobDefinitionID in regroupjobs: regroupjobs[job.jobDefinitionID].append(job) else: regroupjobs[job.jobDefinitionID] = [job] jobgroups = [] ## here converting the grouping into proper JobGroup-Jobs for jobdef in regroupjobs: jobgroup = blocks.get(jobdef, None) if jobgroup is None: configreq = { 'subresource': 'jobgroup', 'subjobdef': jobdef, 'subuser': kwargs['task']['tm_user_dn'] } self.logger.debug( "Retrieving %d jobdef information from task manager db: %s" % (jobdef, str(configreq))) jobgroup = self.server.get(self.resturl, data=configreq) self.logger.debug("Jobgroup information in task manager: %s" % str(jobgroup)) jobgroup = jobgroup[0]['result'][0] blocks[jobdef] = jobgroup['tm_data_blocks'] jg = WMJobGroup() for job in regroupjobs[jobdef]: parser = PassThroughOptionParser() parser.add_option('--inputFile', dest='inputfiles', type='string') parser.add_option('--runAndLumis', dest='runlumis', type='string') parser.add_option('--availableSites', dest='allsites', type='string') parser.add_option('--jobNumber', dest='jobnum', type='int') (options, args) = parser.parse_args(shlex.split(job.jobParameters)) jj = WMJob() jj['input_files'] = [] for infile in literal_eval(options.inputfiles): jj['input_files'].append({ 'lfn': infile, 'block': blocks[jobdef], 'locations': [ss for ss in literal_eval(options.allsites)] }) if options.runlumis: jj['mask']['runAndLumis'] = literal_eval(options.runlumis) jj['panda_oldjobid'] = job.PandaID jj['jobnum'] = options.jobnum jg.add(jj) setattr(jg, 'blocks', blocks[jobdef]) jg.commit() jobgroups.append(jg) return Result(task=kwargs['task'], result=jobgroups)