def expand_jobs(self, step, phase_config): """ Creates and runs JobSteps for a set of tests, based on a phase config. This phase config comes from a tests.json file that the collection jobstep should generate. This method is then called by the TestsJsonHandler. """ if not phase_config.get("cmd"): raise ArtifactParseError("No cmd attribute") if "{test_names}" not in phase_config["cmd"]: raise ArtifactParseError("No {test_names} in cmd") if "tests" not in phase_config: raise ArtifactParseError("No tests attribute") num_tests = len(phase_config["tests"]) test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug) phase, _ = get_or_create( JobPhase, where={"job": step.job, "project": step.project, "label": phase_config.get("phase") or "Test"}, defaults={"status": Status.queued}, ) db.session.commit() # If there are no tests to run, the phase is done. if num_tests == 0: phase.status = Status.finished phase.result = Result.passed db.session.add(phase) db.session.commit() return # Check for whether a previous run of this task has already # created JobSteps for us, since doing it again would create a # double-sharded build. steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all() if steps: step_shard_counts = [s.data.get("shard_count", 1) for s in steps] if len(set(step_shard_counts)) != 1: raise Exception("Mixed shard counts in phase!") elif len(steps) != step_shard_counts[0]: raise Exception("Shard count incorrect") else: # Create all of the job steps and commit them together. groups = TestsExpander.shard_tests(phase_config["tests"], self.max_shards, test_stats, avg_test_time) steps = [ self._create_jobstep( phase, phase_config["cmd"], phase_config.get("path", ""), weight, test_list, len(groups) ) for weight, test_list in groups ] if len(steps) != len(groups): raise Exception("Didn't create correct number of shards") db.session.commit() # Now that that database transaction is done, we'll do the slow work of # creating jenkins builds. for step in steps: self._create_jenkins_build(step) sync_job_step.delay_if_needed(step_id=step.id.hex, task_id=step.id.hex, parent_task_id=phase.job.id.hex)
def expand_jobs(self, step, phase_config): """ Creates and runs JobSteps for a set of tests, based on a phase config. This phase config comes from a tests.json file that the collection jobstep should generate. This method is then called by the TestsJsonHandler. """ assert phase_config['cmd'] assert '{test_names}' in phase_config['cmd'] assert 'tests' in phase_config num_tests = len(phase_config['tests']) test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug) phase, _ = get_or_create(JobPhase, where={ 'job': step.job, 'project': step.project, 'label': phase_config.get('phase') or 'Test', }, defaults={ 'status': Status.queued }) db.session.commit() # If there are no tests to run, the phase is done. if num_tests == 0: phase.status = Status.finished phase.result = Result.passed db.session.add(phase) db.session.commit() return # Check for whether a previous run of this task has already # created JobSteps for us, since doing it again would create a # double-sharded build. steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all() if steps: step_shard_counts = [s.data.get('shard_count', 1) for s in steps] assert len(set(step_shard_counts)) == 1, "Mixed shard counts in phase!" assert len(steps) == step_shard_counts[0] else: # Create all of the job steps and commit them together. groups = TestsExpander.shard_tests(phase_config['tests'], self.max_shards, test_stats, avg_test_time) steps = [ self._create_jobstep(phase, phase_config['cmd'], phase_config.get('path', ''), weight, test_list, len(groups)) for weight, test_list in groups ] assert len(steps) == len(groups) db.session.commit() # Now that that database transaction is done, we'll do the slow work of # creating jenkins builds. for step in steps: self._create_jenkins_build(step) sync_job_step.delay_if_needed( step_id=step.id.hex, task_id=step.id.hex, parent_task_id=phase.job.id.hex, )
def expand_jobs(self, step, phase_config): """ Creates and runs JobSteps for a set of tests, based on a phase config. This phase config comes from a tests.json file that the collection jobstep should generate. This method is then called by the TestsJsonHandler. """ if not phase_config.get('cmd'): raise ArtifactParseError('No cmd attribute') if '{test_names}' not in phase_config['cmd']: raise ArtifactParseError('No {test_names} in cmd') if 'tests' not in phase_config: raise ArtifactParseError('No tests attribute') num_tests = len(phase_config['tests']) test_stats, avg_test_time = TestsExpander.get_test_stats( self.get_test_stats_from() or step.project.slug) phase, _ = get_or_create(JobPhase, where={ 'job': step.job, 'project': step.project, 'label': phase_config.get('phase') or 'Test', }, defaults={'status': Status.queued}) db.session.commit() # If there are no tests to run, the phase is done. if num_tests == 0: phase.status = Status.finished phase.result = Result.passed db.session.add(phase) db.session.commit() return # Check for whether a previous run of this task has already # created JobSteps for us, since doing it again would create a # double-sharded build. steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all() if steps: step_shard_counts = [s.data.get('shard_count', 1) for s in steps] if len(set(step_shard_counts)) != 1: raise Exception("Mixed shard counts in phase!") elif len(steps) != step_shard_counts[0]: raise Exception("Shard count incorrect") else: # Create all of the job steps and commit them together. groups = shard( phase_config['tests'], self.max_shards, test_stats, avg_test_time, normalize_object_name=TestsExpander._normalize_test_segments) steps = [ self._create_jobstep(phase, phase_config['cmd'], phase_config.get('path', ''), weight, test_list, len(groups), cluster=step.cluster) for weight, test_list in groups ] if len(steps) != len(groups): raise Exception("Didn't create correct number of shards") db.session.commit() # Now that the database transaction is done, we'll do the slow work of # creating jenkins builds. for step in steps: self._create_jenkins_build(step) sync_job_step.delay_if_needed( step_id=step.id.hex, task_id=step.id.hex, parent_task_id=phase.job.id.hex, )