def test_adding_result_completes_work_item(work_db): items = [ WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path_{}".format(idx), "operator_{}".format(idx), idx, (idx, idx), (idx, idx + 1)), ) for idx in range(10) ] for item in items: work_db.add_work_item(item) for idx, item in enumerate(items): assert [r[0] for r in work_db.completed_work_items] == items[:idx] result = ( "job_id_{}".format(idx), WorkResult( output="data_{}".format(idx), test_outcome=TOutcome.KILLED, worker_outcome=WorkerOutcome.NORMAL, diff="diff_{}".format(idx), ), ) work_db.set_result(*result)
def __call__(self): """Call `_run()` and return a `WorkItem` with the results. Returns: A `WorkItem` with the `test_outcome` and `data` fields filled in. """ try: test_result = self._run() if test_result[0]: return WorkItem(test_outcome=TestOutcome.SURVIVED, data=test_result[1]) return WorkItem(test_outcome=TestOutcome.KILLED, data=test_result[1]) except Exception: # pylint: disable=broad-except return WorkItem(test_outcome=TestOutcome.INCOMPETENT, data=traceback.format_exception(*sys.exc_info()))
def test_results(work_db): for idx in range(10): work_db.add_work_item( WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path_{}".format(idx), "operator_{}".format(idx), idx, (idx, idx), (idx, idx + 1)), )) original = [( "job_id_{}".format(idx), WorkResult( output="data_{}".format(idx), test_outcome=TOutcome.KILLED, worker_outcome=WorkerOutcome.NORMAL, diff="diff_{}".format(idx), ), ) for idx in range(10)] for result in original: work_db.set_result(*result) actual = list(work_db.results) assert actual == original
def test_set_multiple_results_works(work_db): work_db.add_work_item( WorkItem.single( "job_id", ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1)))) work_db.set_result( "job_id", WorkResult(output="first result", test_outcome=TOutcome.KILLED, worker_outcome=WorkerOutcome.NORMAL, diff="diff"), ) work_db.set_result( "job_id", WorkResult(output="second result", test_outcome=TOutcome.KILLED, worker_outcome=WorkerOutcome.NORMAL, diff="diff"), ) results = [r for job_id, r in work_db.results if job_id == "job_id"] assert len(results) == 1 assert results[0].output == "second result"
def test_num_work_items(work_db): count = 10 for idx in range(count): work_db.add_work_item( WorkItem('path', 'operator', 0, (0, 0), (0, 1), 'job_id_{}'.format(idx))) assert work_db.num_work_items == count
def test_clear_removes_work_items(work_db): for idx in range(10): work_db.add_work_item( WorkItem('path', 'operator', 0, (0, 0), (0, 1), 'job_id_{}'.format(idx))) work_db.clear() assert work_db.num_work_items == 0
def test_adding_result_clears_pending(work_db): items = [ WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path_{}".format(idx), "operator_{}".format(idx), idx, (idx, idx), (idx, idx + 1)), ) for idx in range(10) ] for item in items: work_db.add_work_item(item) for idx, item in enumerate(items): assert sorted(list(work_db.pending_work_items), key=repr) == sorted(items[idx:], key=repr) result = ( "job_id_{}".format(idx), WorkResult( output="data_{}".format(idx), test_outcome=TOutcome.KILLED, worker_outcome=WorkerOutcome.NORMAL, diff="diff_{}".format(idx), ), ) work_db.set_result(*result)
def init(modules, work_db, config, timeout): """Clear and initialize a work-db with work items. Any existing data in the work-db will be cleared and replaced with entirely new work orders. In particular, this means that any results in the db are removed. Args: modules: iterable of module names to be mutated. work_db: A `WorkDB` instance into which the work orders will be saved. config: The configuration for the new session. timeout: The timeout to apply to the work in the session. """ operators = cosmic_ray.plugins.operator_names() counts = cosmic_ray.counting.count_mutants(modules, operators) work_db.set_config(config=config, timeout=timeout) work_db.clear_work_items() work_db.add_work_items( WorkItem(job_id=uuid.uuid4().hex, module=module.__name__, operator=opname, occurrence=occurrence) for module, ops in counts.items() for opname, count in ops.items() for occurrence in range(count))
def test_num_work_items(work_db): count = 10 for idx in range(count): work_db.add_work_item( WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1)))) assert work_db.num_work_items == count
def test_clear_removes_work_items(work_db): for idx in range(10): work_db.add_work_item( WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1)))) work_db.clear() assert work_db.num_work_items == 0
def format_survival_rate(): """cr-rate Usage: cr-rate Read JSON work-records from stdin and print the survival rate. """ records = (WorkItem(json.loads(line, cls=WorkItemJsonDecoder)) for line in sys.stdin) print('{:.2f}'.format(survival_rate(records)))
def test_clear_work_items_removes_results(work_db): for idx in range(10): work_db.add_work_item( WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1)))) work_db.set_result(f"job_id_{idx}", WorkResult(WorkerOutcome.NORMAL)) work_db.clear() assert work_db.num_results == 0
def test_jobs_with_results_are_not_pending(work_db): work_db.add_work_item( WorkItem('path', 'operator', 0, (0, 0), (0, 1), 'job_id')) work_db.set_result( 'job_id', WorkResult(output='data', test_outcome=TestOutcome.KILLED, worker_outcome=WorkerOutcome.NORMAL, diff='diff')) assert not list(work_db.pending_work_items)
def test_new_work_items_are_pending(work_db): items = [ WorkItem('path_{}'.format(idx), 'operator_{}'.format(idx), idx, (idx, idx), (idx, idx + 1), 'job_id_{}'.format(idx)) for idx in range(10) ] for idx, item in enumerate(items): assert list(work_db.pending_work_items) == items[:idx] work_db.add_work_item(item)
def report_xml(): """cr-xml Usage: cr-xml Print an XML formatted report of test results for continuos integration systems """ records = (WorkItem(json.loads(line, cls=WorkItemJsonDecoder)) for line in sys.stdin) xml_elem = _create_xml_report(records) xml_elem.write(sys.stdout.buffer, encoding='utf-8', xml_declaration=True)
def _record_work_item(self, start_pos, end_pos): self.work_db.add_work_item( WorkItem(job_id=uuid.uuid4().hex, module_path=str(self.module_path), operator_name=self.op_name, occurrence=self.occurrence, start_pos=start_pos, end_pos=end_pos)) self.occurrence += 1
def new_work_item(self, operator_name, job_id): self.count += 1 return WorkItem( module_path="{}.py".format(self.count), operator_name=operator_name, occurrence=self.count, start_pos=(self.count, self.count), end_pos=(self.count + 1, self.count + 1), job_id=job_id, )
def test_clear_work_items_removes_results(work_db): for idx in range(10): work_db.add_work_item( WorkItem('path', 'operator', 0, (0, 0), (0, 1), 'job_id_{}'.format(idx))) work_db.set_result('job_id_{}'.format(idx), WorkResult(WorkerOutcome.NORMAL)) work_db.clear() assert work_db.num_results == 0
def baseline(session_file, force, dump_report): """Runs a baseline execution that executes the test suite over unmutated code. Exits with 0 if the job has exited normally, otherwise 1. """ session_file = Path(session_file) baseline_session_file = session_file.parent / '{}.baseline{}'.format( session_file.stem, session_file.suffix) # Find arbitrary work-item in input session that we can copy. with use_db(session_file) as db: # type: WorkDB try: template = next(iter(db.work_items)) except StopIteration: log.error('No work items in session') sys.exit(ExitCode.DATA_ERR) cfg = db.get_config() if force: try: os.unlink(baseline_session_file) except OSError: pass # Copy input work-item, but tell it to use the no-op operator. Create a new # session containing only this work-item and execute this new session. with use_db(baseline_session_file, mode=WorkDB.Mode.create) as db: db.set_config(cfg) db.add_work_item( WorkItem(module_path=template.module_path, operator_name='core/NoOp', occurrence=0, start_pos=template.start_pos, end_pos=template.end_pos, job_id=template.job_id)) # Run the single-entry session. cosmic_ray.commands.execute(db) result = next(db.results)[1] # type: WorkResult if result.test_outcome == TestOutcome.KILLED: if dump_report: print( "Execution with no mutation gives those following errors:") for line in result.output.split('\n'): print(" >>>", line) sys.exit(1) else: if dump_report: print("Execution with no mutation works fine:") sys.exit(ExitCode.OK)
def test_negative_overlap_back(self, lines): item = WorkItem(module_path='foo.py', operator_name='operator', occurrence=0, start_pos=(2, 0), end_pos=(3, 3), job_id='jobid') context = Context(offset=5, topic='', before='', after='', width=6) assert not _item_in_context(lines, item, context)
def __call__(self, timeout, pending_work, config): purge_queue = config['execution-engine'].get('purge-queue', True) try: results = execute_work_items(timeout, pending_work, config) for result in results: yield WorkItem(result.get()) finally: if purge_queue: APP.control.purge()
def _get_work_item(self, start_pos, end_pos): ret = WorkItem( job_id=uuid.uuid4().hex, module_path=str(self.module_path), operator_name=self.op_name, occurrence=self.occurrence, start_pos=start_pos, end_pos=end_pos) self.occurrence += 1 return ret
def test_work_items(work_db): original = [ WorkItem('path_{}'.format(idx), 'operator_{}'.format(idx), idx, (idx, idx), (idx, idx + 1), 'job_id_{}'.format(idx)) for idx in range(10) ] for item in original: work_db.add_work_item(item) actual = list(work_db.work_items) assert actual == original
def new_work_item(self, operator_name, job_id): self.count += 1 return WorkItem.single( job_id, ResolvedMutationSpec( module_path="{}.py".format(self.count), operator_name=operator_name, occurrence=self.count, start_pos=(self.count, self.count), end_pos=(self.count + 1, self.count + 1), ), )
def test_positive_for_perfect_match(self, lines): item = WorkItem( module_path='foo.py', operator_name='operator', occurrence=0, start_pos=(1, 0), end_pos=(3, 2), job_id='jobid') context = Context( offset=0, topic=' ' * 11, before='', after='', width=0) assert _item_in_context(lines, item, context)
def test_jobs_with_results_are_not_pending(work_db): work_db.add_work_item( WorkItem.single( "job_id", ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1)))) work_db.set_result( "job_id", WorkResult(output="data", test_outcome=TOutcome.KILLED, worker_outcome=WorkerOutcome.NORMAL, diff="diff"), ) assert not list(work_db.pending_work_items)
def worker_task(work_item_dict, timeout, config): """The celery task which performs a single mutation and runs a test suite. This runs `cosmic-ray worker` in a subprocess and returns the results, passing `config` to it via stdin. Args: work_item: A dict describing a WorkItem. timeout: The max length of time to let a test run before it's killed config: The configuration to use for the test execution. Returns: An updated WorkItem """ return execute_work_item(WorkItem(work_item_dict), timeout, config)
def test_new_work_items_are_pending(work_db): items = [ WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path_{}".format(idx), "operator_{}".format(idx), idx, (idx, idx), (idx, idx + 1)), ) for idx in range(10) ] for idx, item in enumerate(items): assert sorted(list(work_db.pending_work_items), key=repr) == sorted(items[:idx], key=repr) work_db.add_work_item(item)
def test_work_items(work_db): original = [ WorkItem.single( f"job_id_{idx}", ResolvedMutationSpec("path_{}".format(idx), "operator_{}".format(idx), idx, (idx, idx), (idx, idx + 1)), ) for idx in range(10) ] for item in original: work_db.add_work_item(item) actual = list(work_db.work_items) assert actual == original
def visit_mutation_site(self, node, _, count): """Adds work items to the WorkDB as mutatable nodes are found. """ self.work_db.add_work_items( WorkItem( job_id=uuid.uuid4().hex, module=self.module.__name__, operator=self.op_name, occurrence=self.occurrence + c, filename=self.module.__file__, line_number=get_line_number(node), col_offset=get_col_offset(node)) for c in range(count)) self.occurrence += count return node