def replay(arguments):
root = Path(".")
with database_open(root) as db:
command = db[lexode.pack((0, "command"))]
command = lexode.unpack(command)
command = dict(command)
seed = command.pop("seed")
random.seed(seed)
command = command.pop("command")
alpha, max_workers = check_tests(root, seed, arguments, command)
with database_open(root) as db:
while True:
uids = (lexode.unpack(k)[1] for k, v in db[lexode.pack([2]):]
if v == b"\x00")
uids = sorted(
uids,
key=functools.partial(mutation_diff_size, db),
reverse=True,
)
if not uids:
log.info("No mutation failures 👍")
sys.exit(0)
while uids:
uid = uids.pop(0)
replay_mutation(db, uid, alpha, seed, max_workers, command)
async def play(loop, arguments):
root = Path(".")
seed = arguments["--randomly-seed"] or int(time.time())
log.info("Using random seed: {}".format(seed))
random.seed(seed)
alpha, max_workers = check_tests(root, seed, arguments)
with database_open(root, recreate=True) as db:
# store arguments used to execute command
if arguments["TEST-COMMAND"]:
command = list(arguments["TEST-COMMAND"])
else:
command = list(PYTEST)
command += arguments["<file-or-directory>"]
command = dict(
command=command,
seed=seed,
)
value = list(command.items())
db[lexode.pack((0, "command"))] = lexode.pack(value)
# let's create mutations!
count = await play_create_mutations(loop, root, db, max_workers,
arguments)
# Let's run tests against mutations!
await play_mutations(loop, db, seed, alpha, count, max_workers,
arguments)
def replay_mutation(db, uid, alpha, seed, max_workers, command):
log.info("* Use Ctrl+C to exit.")
command = list(command)
command.append("--randomly-seed={}".format(seed))
max_workers = 1
if max_workers > 1:
command.append("--numprocesses={}".format(max_workers))
timeout = alpha * 2
while True:
ok = mutation_pass((command, uid, timeout))
if not ok:
mutation_show(uid.hex)
msg = "* Type 'skip' to go to next mutation or just enter to retry."
log.info(msg)
skip = input().startswith("s")
if skip:
db[lexode.pack([2, uid])] = b"\x01"
return
# Otherwise loop to re-test...
else:
del db[lexode.pack([2, uid])]
return
def on_mutations_created(items):
nonlocal total
progress.update()
total += len(items)
for path, delta in items:
# TODO: replace ULID with a content addressable hash.
uid = ULID().to_uuid()
# delta is a compressed unified diff
db[lexode.pack([1, uid])] = lexode.pack([path, delta])
def mutation_pass(args): # TODO: rename
command, uid, timeout = args
command = command + ["--mutation={}".format(uid.hex)]
out = run(command, timeout=timeout, silent=True)
if out == 0:
msg = "no error with mutation: {} ({})"
log.trace(msg, " ".join(command), out)
with database_open(".") as db:
db[lexode.pack([2, uid])] = b"\x00"
return False
else:
# TODO: pass root path...
with database_open(".") as db:
del db[lexode.pack([2, uid])]
return True
def mutation_show(uid):
uid = UUID(hex=uid)
log.info("mutation show {}", uid.hex)
log.info("")
with database_open(".") as db:
path, diff = lexode.unpack(db[lexode.pack([1, uid])])
diff = zstd.decompress(diff).decode("utf8")
terminal256 = pygments.formatters.get_formatter_by_name("terminal256")
python = pygments.lexers.get_lexer_by_name("python")
print(diff)
for line in diff.split("\n"):
if line.startswith("+++"):
delta = colored("+++", "green", attrs=["bold"])
highlighted = pygments.highlight(line[3:], python, terminal256)
log.info(delta + highlighted.rstrip())
elif line.startswith("---"):
delta = colored("---", "red", attrs=["bold"])
highlighted = pygments.highlight(line[3:], python, terminal256)
log.info(delta + highlighted.rstrip())
elif line.startswith("+"):
delta = colored("+", "green", attrs=["bold"])
highlighted = pygments.highlight(line[1:], python, terminal256)
log.info(delta + highlighted.rstrip())
elif line.startswith("-"):
delta = colored("-", "red", attrs=["bold"])
highlighted = pygments.highlight(line[1:], python, terminal256)
log.info(delta + highlighted.rstrip())
else:
highlighted = pygments.highlight(line, python, terminal256)
log.info(highlighted.rstrip())
def mutation_apply(uid):
uid = UUID(hex=uid)
with database_open(".") as db:
path, diff = lexode.unpack(db[lexode.pack([1, uid])])
diff = zstd.decompress(diff).decode("utf8")
with open(path, "r") as f:
source = f.read()
patched = patch(diff, source)
with open(path, "w") as f:
f.write(patched)
def mutation_list():
with database_open(".") as db:
uids = ((lexode.unpack(k)[1], v) for k, v in db[lexode.pack([2]):])
uids = sorted(uids,
key=lambda x: mutation_diff_size(db, x[0]),
reverse=True)
if not uids:
log.info("No mutation failures 👍")
sys.exit(0)
for (uid, type) in uids:
log.info("{}\t{}".format(uid.hex,
"skipped" if type == b"\x01" else ""))
def install_module_loader(uid):
db = LSM(".mutation.okvslite")
mutation_show(uid.hex)
path, diff = lexode.unpack(db[lexode.pack([1, uid])])
diff = zstd.decompress(diff).decode("utf8")
with open(path) as f:
source = f.read()
patched = patch(diff, source)
import imp
components = path[:-3].split("/")
while components:
for pythonpath in sys.path:
filepath = os.path.join(pythonpath, "/".join(components))
filepath += ".py"
ok = os.path.exists(filepath)
if ok:
module_path = ".".join(components)
break
else:
components.pop()
continue
break
if module_path is None:
raise Exception("sys.path oops!")
patched_module = imp.new_module(module_path)
try:
exec(patched, patched_module.__dict__)
except Exception:
# TODO: syntaxerror, do not produce those mutations
exec("", patched_module.__dict__)
sys.modules[module_path] = patched_module
def mutation_diff_size(db, uid):
_, diff = lexode.unpack(db[lexode.pack([1, uid])])
out = len(zstd.decompress(diff))
return out
async def play_mutations(loop, db, seed, alpha, total, max_workers, arguments):
# prepare to run tests against mutations
command = list(arguments["TEST-COMMAND"] or PYTEST)
command.append("--randomly-seed={}".format(seed))
command.extend(arguments["<file-or-directory>"])
eta = humanize(alpha * total / max_workers)
log.success("It will take at most {} to run the mutations", eta)
timeout = alpha * 2
uids = db[lexode.pack([1]):lexode.pack([2])]
uids = ((command, lexode.unpack(key)[1], timeout) for (key, _) in uids)
# sampling
sampling = arguments["--sampling"]
sampler, total = sampling_setup(sampling, total)
uids = sampler(uids)
step = 10
gamma = time.perf_counter()
remaining = total
log.info("Testing mutations in progress...")
with tqdm(total=100) as progress:
def on_progress(_):
nonlocal remaining
nonlocal step
nonlocal gamma
remaining -= 1
if (remaining % step) == 0:
percent = 100 - ((remaining / total) * 100)
now = time.perf_counter()
delta = now - gamma
eta = (delta / step) * remaining
progress.update(int(percent))
progress.set_description("ETA {}".format(humanize(eta)))
msg = "Mutation tests {:.2f}% done..."
log.debug(msg, percent)
log.debug("ETA {}...", humanize(eta))
for speed in [10_000, 1_000, 100, 10, 1]:
if total // speed == 0:
continue
step = speed
break
gamma = time.perf_counter()
with timeit() as delta:
with futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
await pool_for_each_par_map(loop, pool, on_progress,
mutation_pass, uids)
errors = len(list(db[lexode.pack([2]):lexode.pack([3])]))