def test_adding_result_completes_work_item(work_db):
items = [
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path_{}".format(idx),
"operator_{}".format(idx), idx, (idx, idx),
(idx, idx + 1)),
) for idx in range(10)
]
for item in items:
work_db.add_work_item(item)
for idx, item in enumerate(items):
assert [r[0] for r in work_db.completed_work_items] == items[:idx]
result = (
"job_id_{}".format(idx),
WorkResult(
output="data_{}".format(idx),
test_outcome=TOutcome.KILLED,
worker_outcome=WorkerOutcome.NORMAL,
diff="diff_{}".format(idx),
),
)
work_db.set_result(*result)
def __call__(self):
"""Call `_run()` and return a `WorkItem` with the results.
Returns: A `WorkItem` with the `test_outcome` and `data` fields
filled in.
"""
try:
test_result = self._run()
if test_result[0]:
return WorkItem(test_outcome=TestOutcome.SURVIVED,
data=test_result[1])
return WorkItem(test_outcome=TestOutcome.KILLED,
data=test_result[1])
except Exception: # pylint: disable=broad-except
return WorkItem(test_outcome=TestOutcome.INCOMPETENT,
data=traceback.format_exception(*sys.exc_info()))
def test_results(work_db):
for idx in range(10):
work_db.add_work_item(
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path_{}".format(idx),
"operator_{}".format(idx), idx,
(idx, idx), (idx, idx + 1)),
))
original = [(
"job_id_{}".format(idx),
WorkResult(
output="data_{}".format(idx),
test_outcome=TOutcome.KILLED,
worker_outcome=WorkerOutcome.NORMAL,
diff="diff_{}".format(idx),
),
) for idx in range(10)]
for result in original:
work_db.set_result(*result)
actual = list(work_db.results)
assert actual == original
def test_set_multiple_results_works(work_db):
work_db.add_work_item(
WorkItem.single(
"job_id",
ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1))))
work_db.set_result(
"job_id",
WorkResult(output="first result",
test_outcome=TOutcome.KILLED,
worker_outcome=WorkerOutcome.NORMAL,
diff="diff"),
)
work_db.set_result(
"job_id",
WorkResult(output="second result",
test_outcome=TOutcome.KILLED,
worker_outcome=WorkerOutcome.NORMAL,
diff="diff"),
)
results = [r for job_id, r in work_db.results if job_id == "job_id"]
assert len(results) == 1
assert results[0].output == "second result"
def test_num_work_items(work_db):
count = 10
for idx in range(count):
work_db.add_work_item(
WorkItem('path', 'operator', 0, (0, 0), (0, 1),
'job_id_{}'.format(idx)))
assert work_db.num_work_items == count
def test_clear_removes_work_items(work_db):
for idx in range(10):
work_db.add_work_item(
WorkItem('path', 'operator', 0, (0, 0), (0, 1),
'job_id_{}'.format(idx)))
work_db.clear()
assert work_db.num_work_items == 0
def test_adding_result_clears_pending(work_db):
items = [
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path_{}".format(idx),
"operator_{}".format(idx), idx, (idx, idx),
(idx, idx + 1)),
) for idx in range(10)
]
for item in items:
work_db.add_work_item(item)
for idx, item in enumerate(items):
assert sorted(list(work_db.pending_work_items),
key=repr) == sorted(items[idx:], key=repr)
result = (
"job_id_{}".format(idx),
WorkResult(
output="data_{}".format(idx),
test_outcome=TOutcome.KILLED,
worker_outcome=WorkerOutcome.NORMAL,
diff="diff_{}".format(idx),
),
)
work_db.set_result(*result)
def init(modules, work_db, config, timeout):
"""Clear and initialize a work-db with work items.
Any existing data in the work-db will be cleared and replaced with entirely
new work orders. In particular, this means that any results in the db are
removed.
Args:
modules: iterable of module names to be mutated.
work_db: A `WorkDB` instance into which the work orders will be saved.
config: The configuration for the new session.
timeout: The timeout to apply to the work in the session.
"""
operators = cosmic_ray.plugins.operator_names()
counts = cosmic_ray.counting.count_mutants(modules, operators)
work_db.set_config(config=config, timeout=timeout)
work_db.clear_work_items()
work_db.add_work_items(
WorkItem(job_id=uuid.uuid4().hex,
module=module.__name__,
operator=opname,
occurrence=occurrence) for module, ops in counts.items()
for opname, count in ops.items() for occurrence in range(count))
def test_num_work_items(work_db):
count = 10
for idx in range(count):
work_db.add_work_item(
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1))))
assert work_db.num_work_items == count
def test_clear_removes_work_items(work_db):
for idx in range(10):
work_db.add_work_item(
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1))))
work_db.clear()
assert work_db.num_work_items == 0
def format_survival_rate():
"""cr-rate
Usage: cr-rate
Read JSON work-records from stdin and print the survival rate.
"""
records = (WorkItem(json.loads(line, cls=WorkItemJsonDecoder)) for line in sys.stdin)
print('{:.2f}'.format(survival_rate(records)))
def test_clear_work_items_removes_results(work_db):
for idx in range(10):
work_db.add_work_item(
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1))))
work_db.set_result(f"job_id_{idx}", WorkResult(WorkerOutcome.NORMAL))
work_db.clear()
assert work_db.num_results == 0
def test_jobs_with_results_are_not_pending(work_db):
work_db.add_work_item(
WorkItem('path', 'operator', 0, (0, 0), (0, 1), 'job_id'))
work_db.set_result(
'job_id',
WorkResult(output='data',
test_outcome=TestOutcome.KILLED,
worker_outcome=WorkerOutcome.NORMAL,
diff='diff'))
assert not list(work_db.pending_work_items)
def test_new_work_items_are_pending(work_db):
items = [
WorkItem('path_{}'.format(idx), 'operator_{}'.format(idx), idx,
(idx, idx), (idx, idx + 1), 'job_id_{}'.format(idx))
for idx in range(10)
]
for idx, item in enumerate(items):
assert list(work_db.pending_work_items) == items[:idx]
work_db.add_work_item(item)
def report_xml():
"""cr-xml
Usage: cr-xml
Print an XML formatted report of test results for continuos integration systems
"""
records = (WorkItem(json.loads(line, cls=WorkItemJsonDecoder)) for line in sys.stdin)
xml_elem = _create_xml_report(records)
xml_elem.write(sys.stdout.buffer, encoding='utf-8', xml_declaration=True)
def _record_work_item(self, start_pos, end_pos):
self.work_db.add_work_item(
WorkItem(job_id=uuid.uuid4().hex,
module_path=str(self.module_path),
operator_name=self.op_name,
occurrence=self.occurrence,
start_pos=start_pos,
end_pos=end_pos))
self.occurrence += 1
def new_work_item(self, operator_name, job_id):
self.count += 1
return WorkItem(
module_path="{}.py".format(self.count),
operator_name=operator_name,
occurrence=self.count,
start_pos=(self.count, self.count),
end_pos=(self.count + 1, self.count + 1),
job_id=job_id,
)
def test_clear_work_items_removes_results(work_db):
for idx in range(10):
work_db.add_work_item(
WorkItem('path', 'operator', 0, (0, 0), (0, 1),
'job_id_{}'.format(idx)))
work_db.set_result('job_id_{}'.format(idx),
WorkResult(WorkerOutcome.NORMAL))
work_db.clear()
assert work_db.num_results == 0
def baseline(session_file, force, dump_report):
"""Runs a baseline execution that executes the test suite over unmutated code.
Exits with 0 if the job has exited normally, otherwise 1.
"""
session_file = Path(session_file)
baseline_session_file = session_file.parent / '{}.baseline{}'.format(
session_file.stem, session_file.suffix)
# Find arbitrary work-item in input session that we can copy.
with use_db(session_file) as db: # type: WorkDB
try:
template = next(iter(db.work_items))
except StopIteration:
log.error('No work items in session')
sys.exit(ExitCode.DATA_ERR)
cfg = db.get_config()
if force:
try:
os.unlink(baseline_session_file)
except OSError:
pass
# Copy input work-item, but tell it to use the no-op operator. Create a new
# session containing only this work-item and execute this new session.
with use_db(baseline_session_file, mode=WorkDB.Mode.create) as db:
db.set_config(cfg)
db.add_work_item(
WorkItem(module_path=template.module_path,
operator_name='core/NoOp',
occurrence=0,
start_pos=template.start_pos,
end_pos=template.end_pos,
job_id=template.job_id))
# Run the single-entry session.
cosmic_ray.commands.execute(db)
result = next(db.results)[1] # type: WorkResult
if result.test_outcome == TestOutcome.KILLED:
if dump_report:
print(
"Execution with no mutation gives those following errors:")
for line in result.output.split('\n'):
print(" >>>", line)
sys.exit(1)
else:
if dump_report:
print("Execution with no mutation works fine:")
sys.exit(ExitCode.OK)
def test_negative_overlap_back(self, lines):
item = WorkItem(module_path='foo.py',
operator_name='operator',
occurrence=0,
start_pos=(2, 0),
end_pos=(3, 3),
job_id='jobid')
context = Context(offset=5, topic='', before='', after='', width=6)
assert not _item_in_context(lines, item, context)
def __call__(self, timeout, pending_work, config):
purge_queue = config['execution-engine'].get('purge-queue', True)
try:
results = execute_work_items(timeout, pending_work, config)
for result in results:
yield WorkItem(result.get())
finally:
if purge_queue:
APP.control.purge()
def _get_work_item(self, start_pos, end_pos):
ret = WorkItem(
job_id=uuid.uuid4().hex,
module_path=str(self.module_path),
operator_name=self.op_name,
occurrence=self.occurrence,
start_pos=start_pos,
end_pos=end_pos)
self.occurrence += 1
return ret
def test_work_items(work_db):
original = [
WorkItem('path_{}'.format(idx), 'operator_{}'.format(idx), idx,
(idx, idx), (idx, idx + 1), 'job_id_{}'.format(idx))
for idx in range(10)
]
for item in original:
work_db.add_work_item(item)
actual = list(work_db.work_items)
assert actual == original
def new_work_item(self, operator_name, job_id):
self.count += 1
return WorkItem.single(
job_id,
ResolvedMutationSpec(
module_path="{}.py".format(self.count),
operator_name=operator_name,
occurrence=self.count,
start_pos=(self.count, self.count),
end_pos=(self.count + 1, self.count + 1),
),
)
def test_positive_for_perfect_match(self, lines):
item = WorkItem(
module_path='foo.py',
operator_name='operator',
occurrence=0,
start_pos=(1, 0),
end_pos=(3, 2),
job_id='jobid')
context = Context(
offset=0, topic=' ' * 11, before='', after='', width=0)
assert _item_in_context(lines, item, context)
def test_jobs_with_results_are_not_pending(work_db):
work_db.add_work_item(
WorkItem.single(
"job_id",
ResolvedMutationSpec("path", "operator", 0, (0, 0), (0, 1))))
work_db.set_result(
"job_id",
WorkResult(output="data",
test_outcome=TOutcome.KILLED,
worker_outcome=WorkerOutcome.NORMAL,
diff="diff"),
)
assert not list(work_db.pending_work_items)
def worker_task(work_item_dict, timeout, config):
"""The celery task which performs a single mutation and runs a test suite.
This runs `cosmic-ray worker` in a subprocess and returns the results,
passing `config` to it via stdin.
Args:
work_item: A dict describing a WorkItem.
timeout: The max length of time to let a test run before it's killed
config: The configuration to use for the test execution.
Returns: An updated WorkItem
"""
return execute_work_item(WorkItem(work_item_dict), timeout, config)
def test_new_work_items_are_pending(work_db):
items = [
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path_{}".format(idx),
"operator_{}".format(idx), idx, (idx, idx),
(idx, idx + 1)),
) for idx in range(10)
]
for idx, item in enumerate(items):
assert sorted(list(work_db.pending_work_items),
key=repr) == sorted(items[:idx], key=repr)
work_db.add_work_item(item)
def test_work_items(work_db):
original = [
WorkItem.single(
f"job_id_{idx}",
ResolvedMutationSpec("path_{}".format(idx),
"operator_{}".format(idx), idx, (idx, idx),
(idx, idx + 1)),
) for idx in range(10)
]
for item in original:
work_db.add_work_item(item)
actual = list(work_db.work_items)
assert actual == original
def visit_mutation_site(self, node, _, count):
"""Adds work items to the WorkDB as mutatable nodes are found.
"""
self.work_db.add_work_items(
WorkItem(
job_id=uuid.uuid4().hex,
module=self.module.__name__,
operator=self.op_name,
occurrence=self.occurrence + c,
filename=self.module.__file__,
line_number=get_line_number(node),
col_offset=get_col_offset(node))
for c in range(count))
self.occurrence += count
return node
