def buildDAG(args, modified_files, formula_dir): oDag = dag.DAG() dMap = {} common_names = set([]) for directory, d_list, f_list in os.walk(formula_dir): if 'meta.yaml' in f_list: meta = MetaData(os.path.join(directory, 'meta.yaml')) reqs = meta.meta['requirements'] combined_deps = set(reqs.get('build', '')).union(reqs.get('run', '')) common_names.add(meta.name()) dMap[meta.meta_path] = (meta.name(), combined_deps, meta) # Populate DAG [oDag.add_node(x) for x in dMap.keys()] # Create edges for ind_node, name, dependencies, meta, dag_node in _walkMapAndDag(dMap, oDag): controlled_dependencies = set(dependencies).intersection(common_names) if dMap[dag_node][0] in controlled_dependencies: oDag.add_edge(dag_node, ind_node) # Remove edges (skips, unmodified recipes, etc) for ind_node, name, dependencies, meta, dag_node in _walkMapAndDag(dMap, oDag): controlled_dependencies = set(dependencies).intersection(common_names) if ind_node not in modified_files and controlled_dependencies and args.dependencies: continue elif ind_node not in modified_files: oDag.delete_node_if_exists(ind_node) return oDag
def schedule(self, testers): """ Schedule supplied list of testers for execution. """ # If any threads caused an exception, we have already closed down the queue and need to # not schedule any more jobs if self.run_pool._state: return # Instance the DAG class so we can share it amongst all the TesterData containers job_dag = dag.DAG() non_runnable_jobs = set([]) name_to_job_container = {} # Increment our simple queue count with the number of testers the scheduler received with self.slot_lock: self.job_queue_count += len(testers) # Create a local dictionary of tester names to job containers. Add this dictionary to a # set. We will use this set as a way to gain access to their methods. for tester in testers: name_to_job_container[tester.getTestName()] = TesterData(tester, job_dag, self.options) self.tester_datas.add(name_to_job_container[tester.getTestName()]) # Populate job_dag with testers. This method will also return any testers which caused failures # while building the DAG. skipped_or_failed_testers = self.buildDAG(name_to_job_container, job_dag) # Create a set of failing job containers for failed_tester in skipped_or_failed_testers: non_runnable_jobs.add(name_to_job_container[failed_tester.getTestName()]) # Iterate over the jobs in our non_runnable_jobs and handle any downstream jobs affected by # 'job'. These will be our 'skipped dependency' tests. for job in non_runnable_jobs.copy(): additionally_skipped = self.processDownstreamTests(job) non_runnable_jobs.update(additionally_skipped) job_dag.delete_node_if_exists(job) # Get a count of all the items still in the DAG. These will be the jobs that ultimately are queued runnable_jobs = job_dag.size() # Make sure we didn't drop a tester somehow if len(non_runnable_jobs) + runnable_jobs != len(testers): raise SchedulerError('Runnable tests in addition to Skipped tests does not match total scheduled test count!') # Assign a status thread to begin work on any skipped/failed jobs self.queueJobs(status_jobs=non_runnable_jobs) # Build our list of runnable jobs and set the tester's status to queued job_list = [] if runnable_jobs: job_list = job_dag.ind_nodes() for job_container in job_list: tester = job_container.getTester() tester.setStatus('QUEUED', tester.bucket_pending) # Queue runnable jobs self.queueJobs(run_jobs=job_list)
def removeAllDependencies(self): """ Flatten current DAG so that it no longer contains any dependency information """ if self.__name_to_job and self.__job_dag.size(): tmp_job_dag = dag.DAG() for job in self.__job_dag.topological_sort(): tmp_job_dag.add_node(job) self.__job_dag = tmp_job_dag return self.__job_dag
def buildDAG(template_dir, args): package_dag = dag.DAG() for template_file in os.listdir(template_dir): tmp_job = Job(args, package_file=os.path.join(template_dir, template_file), name=template_file) package_dag.add_node(tmp_job) return buildEdges(package_dag, args)
def schedule(self, dag_object): """ Account for all the jobs in the DAG and submit the DAG for processing. """ if type(dag_object) != type(dag.DAG()): raise SchedulerError('schedule method requires a DAG object') if self.worker_pool._state: raise SchedulerError('scheduler is no longer accepting jobs') with self.thread_lock: self.job_queue_count += dag_object.size() for job in dag_object.topological_sort(): # create pointer to DAG for every job self.shared_dags[job] = JobContainer(dag_object) self.queueDAG(dag_object)
def schedule(self, testers): """ Schedule supplied list of testers for execution. """ # If any threads caused an exception, we have already closed down the queue and need to # not schedule any more jobs if self.error_state: return # Instance the DAG class so we can share it amongst all the Job containers job_dag = dag.DAG() runnable_jobs = set([]) non_runnable_jobs = set([]) name_to_job_container = {} # Increment our simple queue count with the number of testers the scheduler received with self.job_queue_lock: self.job_queue_count += len(testers) # Create a local dictionary of tester names to job containers. Add this dictionary to a # set. We will use this set as a way to gain access to their methods. for tester in testers: if tester.getTestName() in name_to_job_container: tester.addCaveats('duplicate test') tester.setStatus(tester.bucket_skip.status, tester.bucket_skip) non_runnable_jobs.add(Job(tester, job_dag, self.options)) else: name_to_job_container[tester.getTestName()] = Job(tester, job_dag, self.options) self.tester_datas.add(name_to_job_container[tester.getTestName()]) # Populate job_dag with testers. This method will also return any testers which caused failures # while building the DAG. skipped_or_failed_testers = self.buildDAG(name_to_job_container, job_dag) # Create a set of failing job containers for failed_tester in skipped_or_failed_testers: non_runnable_jobs.add(name_to_job_container[failed_tester.getTestName()]) # Iterate over the jobs in our non_runnable_jobs and handle any downstream jobs affected by # 'job'. These will be our 'skipped dependency' tests. for job in non_runnable_jobs.copy(): additionally_skipped = self.processDownstreamTests(job) non_runnable_jobs.update(additionally_skipped) job_dag.delete_node_if_exists(job) # Make sure we didn't drop a tester somehow if len(non_runnable_jobs) + job_dag.size() != len(testers): raise SchedulerError('Runnable tests in addition to Skipped tests does not match total scheduled test count!') # Inform derived schedulers of the jobs we are skipping immediately self.reportSkipped(non_runnable_jobs) # Allow derived schedulers to modify the dag before we launch # TODO: We don't like this, and this will change when we move to better DAG handling. if job_dag.size(): self.preLaunch(job_dag) # Build our list of runnable jobs and set the tester's status to queued if job_dag.size(): runnable_jobs = job_dag.ind_nodes() for job in runnable_jobs: tester = job.getTester() tester.setStatus('QUEUED', tester.bucket_pending) # Queue generated jobs self.queueJobs(run_jobs=runnable_jobs, status_jobs=non_runnable_jobs)
def __init__(self, options): self.__job_dag = dag.DAG() self.options = options
def __init__(self, options): self.__job_dag = dag.DAG() self.__parallel_scheduling = None self.options = options
def buildDAG(args, modified_files, formula_dir): formula_dag = dag.DAG() for formula_file in os.listdir(formula_dir): if formula_file.endswith(".rb"): formula_dag.add_node(os.path.basename(formula_file)[:-3]) return buildEdges(args, modified_files, formula_dir, formula_dag)
def buildDAG(args, modified_files, formula_dir): formula_dag = dag.DAG() for directory, d_list, f_list in os.walk(formula_dir): if 'meta.yaml' in f_list: formula_dag.add_node(os.path.join(directory)) return buildEdges(args, modified_files, formula_dir, formula_dag)