def __call__(self, parameters): """ Poller for looking in all active subscriptions for jobs that need to be made. """ logging.info("In JobCreatorWorker.__call__") myThread = threading.currentThread() for entry in parameters: # This retrieves a single subscription subscriptionID = entry.get('subscription') if subscriptionID < 0: logging.error("Got non-existant subscription") logging.error("Assuming parameters in error: returning") return subscriptionID myThread.transaction.begin() logging.info("About to call subscription %i" %subscriptionID) wmbsSubscription = Subscription(id = subscriptionID) wmbsSubscription.load() wmbsSubscription["workflow"].load() workflow = wmbsSubscription["workflow"] wmWorkload = retrieveWMSpec(wmbsSubscription) if not workflow.task or not wmWorkload: # Then we have a problem # We have no sandbox # We NEED a sandbox # Abort this subscription! # But do NOT fail # We have no way of marking a subscription as bad per se # We'll have to just keep skipping it wmTask = None seederList = [] logging.error("Have no task for workflow %i" % (workflow.id)) logging.error("Aborting Subscription %i" % (subscriptionID)) continue else: wmTask = wmWorkload.getTaskByPath(workflow.task) if hasattr(wmTask.data, 'seeders'): manager = SeederManager(wmTask) seederList = manager.getSeederList() else: seederList = [] logging.info("About to enter JobFactory") logging.debug("Going to call wmbsJobFactory with limit %i" % (self.limit)) # My hope is that the job factory is smart enough only to split un-split jobs wmbsJobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = wmbsSubscription, generators=seederList, limit = self.limit) splitParams = retrieveJobSplitParams(wmWorkload, workflow.task) logging.debug("Split Params: %s" % splitParams) continueSubscription = True myThread.transaction.commit() # Turn on the jobFactory myThread.transaction.begin() wmbsJobFactory.open() # Create a function to hold it jobSplittingFunction = runSplitter(jobFactory = wmbsJobFactory, splitParams = splitParams) while continueSubscription: # This loop runs over the jobFactory, # using yield statements and a pre-existing proxy to # generate and process new jobs # First we need the jobs. try: wmbsJobGroups = next(jobSplittingFunction) logging.info("Retrieved %i jobGroups from jobSplitter" % (len(wmbsJobGroups))) except StopIteration: # If you receive a stopIteration, we're done logging.info("Completed iteration over subscription %i" % (subscriptionID)) continueSubscription = False continue # Now we get to find out what job they are. countJobs = self.daoFactory(classname = "Jobs.GetNumberOfJobsPerWorkflow") jobNumber = countJobs.execute(workflow = workflow.id, conn = myThread.transaction.conn, transaction = True) logging.debug("Have %i jobs for this workflow already" % (jobNumber)) for wmbsJobGroup in wmbsJobGroups: logging.debug("Processing jobGroup %i" % (wmbsJobGroup.exists())) logging.debug("Processing %i jobs" % (len(wmbsJobGroup.jobs)) ) # Create a directory self.createWorkArea.processJobs(jobGroup = wmbsJobGroup, startDir = self.jobCacheDir, workflow = workflow, wmWorkload = wmWorkload, transaction = myThread.transaction, conn = myThread.transaction.conn) for job in wmbsJobGroup.jobs: jobNumber += 1 self.saveJob(job = job, workflow = workflow, wmTask = wmTask, jobNumber = jobNumber) self.advanceJobGroup(wmbsJobGroup) logging.debug("Finished call for jobGroup %i" \ % (wmbsJobGroup.exists())) # END: while loop over jobSplitter myThread.transaction.commit() # About to reset everything wmbsJobGroups = None wmTask = None wmWorkload = None splitParams = None wmbsJobFactory = None gc.collect() # About to check memory doMemoryCheck("About to get memory references: End of subscription loop") # Final memory check doMemoryCheck("About to get memory references: End of __call__()") logging.debug("About to return from JobCreatorWorker.__call__()") return parameters