def handle_no_cache(context):
"""Handle lack-of-cache error, prompt user for index process."""
logger.error(
_("Could not locate wily cache, the cache is required to provide insights.")
)
p = input(_("Do you want to run setup and index your project now? [y/N]"))
if p.lower() != "y":
exit(1)
else:
revisions = input(_("How many previous git revisions do you want to index? : "))
revisions = int(revisions)
path = input(_("Path to your source files; comma-separated for multiple: "))
paths = path.split(",")
context.invoke(build, max_revisions=revisions, targets=paths, operators=None)
def rank(config, path, metric, revision_index, limit, threshold, descending):
"""
Rank command ordering files, methods or functions using metrics.
:param config: The configuration
:type config: :class:'wily.config.WilyConfig'
:param path: The path to the file
:type path ''str''
:param metric: Name of the metric to report on
:type metric: ''str''
:param revision_index: Version of git repository to revert to.
:type revision_index: ``str``
:param limit: Limit the number of items in the table
:type limit: ``int``
:param threshold: For total values beneath the threshold return a non-zero exit code
:type threshold: ``int``
:return: Sorted table of all files in path, sorted in order of metric.
"""
logger.debug("Running rank command")
data = []
operator, metric = resolve_metric_as_tuple(metric)
operator = operator.name
state = State(config)
if not revision_index:
target_revision = state.index[state.default_archiver].last_revision
else:
rev = resolve_archiver(
state.default_archiver).cls(config).find(revision_index)
logger.debug(f"Resolved {revision_index} to {rev.key} ({rev.message})")
try:
target_revision = state.index[state.default_archiver][rev.key]
except KeyError:
logger.error(
f"Revision {revision_index} is not in the cache, make sure you have run wily build."
)
exit(1)
logger.info(
f"-----------Rank for {metric.description} for {format_revision(target_revision.revision.key)} by {target_revision.revision.author_name} on {format_date(target_revision.revision.date)}.------------"
)
if path is None:
files = target_revision.get_paths(config, state.default_archiver,
operator)
logger.debug(f"Analysing {files}")
else:
# Resolve target paths when the cli has specified --path
if config.path != DEFAULT_PATH:
targets = [str(Path(config.path) / Path(path))]
else:
targets = [path]
# Expand directories to paths
files = [
os.path.relpath(fn, config.path)
for fn in radon.cli.harvest.iter_filenames(targets)
]
logger.debug(f"Targeting - {files}")
for item in files:
for archiver in state.archivers:
try:
logger.debug(
f"Fetching metric {metric.name} for {operator} in {str(item)}"
)
val = target_revision.get(config, archiver, operator,
str(item), metric.name)
value = val
data.append((item, value))
except KeyError:
logger.debug(f"Could not find file {item} in index")
# Sort by ideal value
data = sorted(data, key=op.itemgetter(1), reverse=descending)
if limit:
data = data[:limit]
# Tack on the total row at the end
total = metric.aggregate(rev[1] for rev in data)
data.append(["Total", total])
headers = ("File", metric.description)
print(
tabulate.tabulate(headers=headers,
tabular_data=data,
tablefmt=DEFAULT_GRID_STYLE))
if threshold and total < threshold:
logger.error(
f"Total value below the specified threshold: {total} < {threshold}"
)
exit(1)
def build(config, archiver, operators):
"""
Build the history given a archiver and collection of operators.
:param config: The wily configuration
:type config: :namedtuple:`wily.config.WilyConfig`
:param archiver: The archiver to use
:type archiver: :namedtuple:`wily.archivers.Archiver`
:param operators: The list of operators to execute
:type operators: `list` of :namedtuple:`wily.operators.Operator`
"""
try:
logger.debug(f"Using {archiver.name} archiver module")
archiver = archiver.cls(config)
revisions = archiver.revisions(config.path, config.max_revisions)
except InvalidGitRepositoryError:
# TODO: This logic shouldn't really be here (SoC)
logger.info(f"Defaulting back to the filesystem archiver, not a valid git repo")
archiver = FilesystemArchiver(config)
revisions = archiver.revisions(config.path, config.max_revisions)
except Exception as e:
if hasattr(e, "message"):
logger.error(f"Failed to setup archiver: '{e.message}'")
else:
logger.error(f"Failed to setup archiver: '{type(e)} - {e}'")
exit(1)
state = State(config, archiver=archiver)
# Check for existence of cache, else provision
state.ensure_exists()
index = state.index[archiver.name]
# remove existing revisions from the list
revisions = [revision for revision in revisions if revision not in index][::-1]
logger.info(
f"Found {len(revisions)} revisions from '{archiver.name}' archiver in '{config.path}'."
)
_op_desc = ",".join([operator.name for operator in operators])
logger.info(f"Running operators - {_op_desc}")
bar = Bar("Processing", max=len(revisions) * len(operators))
state.operators = operators
# Index all files the first time, only scan changes afterward
seed = True
prev_roots = None
try:
with multiprocessing.Pool(processes=len(operators)) as pool:
for revision in revisions:
# Checkout target revision
archiver.checkout(revision, config.checkout_options)
stats = {"operator_data": {}}
if seed:
targets = config.targets
else: # Only target changed files
# TODO : Check that changed files are children of the targets
targets = [
str(pathlib.Path(config.path) / pathlib.Path(file))
for file in revision.files
# if any([True for target in config.targets if
# target in pathlib.Path(pathlib.Path(config.path) / pathlib.Path(file)).parents])
]
# Run each operator as a separate process
data = pool.starmap(
run_operator,
[(operator, revision, config, targets) for operator in operators],
)
# data is a list of tuples, where for each operator, it is a tuple of length 2,
operator_data_len = 2
# second element in the tuple, i.e data[i][1]) has the collected data
for i in range(0, len(operators)):
if i < len(data) and len(data[i]) >= operator_data_len and len(data[i][1]) == 0:
logger.warn(f"In revision {revision.key}, for operator {operators[i].name}: No data collected")
# Map the data back into a dictionary
for operator_name, result in data:
# find all unique directories in the results
roots = {pathlib.Path(entry).parents[0] for entry in result.keys()}
indices = set(result.keys())
# For a seed run, there is no previous change set, so use current
if seed:
prev_roots = roots
prev_indices = indices
roots = prev_roots | roots
# Copy the ir from any unchanged files from the prev revision
if not seed:
missing_indices = prev_indices - indices
# TODO: Check existence of file path.
for missing in missing_indices:
# Don't copy aggregate keys as their values may have changed
if missing in roots:
continue
# previous index may not have that operator
if operator_name not in prev_stats["operator_data"]:
continue
# previous index may not have file either
if (
missing
not in prev_stats["operator_data"][operator_name]
):
continue
result[missing] = prev_stats["operator_data"][
operator_name
][missing]
# Aggregate metrics across all root paths using the aggregate function in the metric
for root in roots:
# find all matching entries recursively
aggregates = [
path
for path in result.keys()
if root in pathlib.Path(path).parents
]
result[str(root)] = {"total": {}}
# aggregate values
for metric in resolve_operator(operator_name).cls.metrics:
func = metric.aggregate
values = [
result[aggregate]["total"][metric.name]
for aggregate in aggregates
if aggregate in result
and metric.name in result[aggregate]["total"]
]
if len(values) > 0:
result[str(root)]["total"][metric.name] = func(values)
prev_indices = set(result.keys())
prev_roots = roots
stats["operator_data"][operator_name] = result
bar.next()
prev_stats = stats
seed = False
ir = index.add(revision, operators=operators)
ir.store(config, archiver, stats)
index.save()
bar.finish()
except Exception as e:
logger.error(f"Failed to build cache: {type(e)}: '{e}'")
raise e
finally:
# Reset the archive after every run back to the head of the branch
archiver.finish()
"""Handle lack-of-cache error, prompt user for index process."""
logger.error(
f"Could not locate wily cache, the cache is required to provide insights."
)
p = input("Do you want to run setup and index your project now? [y/N]")
if p.lower() != "y":
exit(1)
else:
revisions = input(
"How many previous git revisions do you want to index? : ")
revisions = int(revisions)
path = input(
"Path to your source files; comma-separated for multiple: ")
paths = path.split(",")
context.invoke(build,
max_revisions=revisions,
targets=paths,
operators=None)
if __name__ == "__main__": # pragma: no cover
try:
cli()
except Exception as runtime:
logger.error(
f"Oh no, Wily crashed! See {WILY_LOG_NAME} for information.")
logger.info(
f"If you think this crash was unexpected, please raise an issue at https://github.com/tonybaloney/wily/issues and copy the log file into the issue report along with some information on what you were doing."
)
logger.debug(traceback.format_exc())
def diff(config,
files,
metrics,
changes_only=True,
detail=True,
revision=None):
"""
Show the differences in metrics for each of the files.
:param config: The wily configuration
:type config: :namedtuple:`wily.config.WilyConfig`
:param files: The files to compare.
:type files: ``list`` of ``str``
:param metrics: The metrics to measure.
:type metrics: ``list`` of ``str``
:param changes_only: Only include changes files in output.
:type changes_only: ``bool``
:param detail: Show details (function-level)
:type detail: ``bool``
:param revision: Compare with specific revision
:type revision: ``str``
"""
config.targets = files
files = list(files)
state = State(config)
# Resolve target paths when the cli has specified --path
if config.path != DEFAULT_PATH:
targets = [str(Path(config.path) / Path(file)) for file in files]
else:
targets = files
# Expand directories to paths
files = [
os.path.relpath(fn, config.path)
for fn in radon.cli.harvest.iter_filenames(targets)
]
logger.debug(f"Targeting - {files}")
if not revision:
target_revision = state.index[state.default_archiver].last_revision
else:
rev = resolve_archiver(
state.default_archiver).cls(config).find(revision)
logger.debug(f"Resolved {revision} to {rev.key} ({rev.message})")
try:
target_revision = state.index[state.default_archiver][rev.key]
except KeyError:
logger.error(
f"Revision {revision} is not in the cache, make sure you have run wily build."
)
exit(1)
logger.info(
f"Comparing current with {format_revision(target_revision.revision.key)} by {target_revision.revision.author_name} on {format_date(target_revision.revision.date)}."
)
# Convert the list of metrics to a list of metric instances
operators = {resolve_operator(metric.split(".")[0]) for metric in metrics}
metrics = [(metric.split(".")[0], resolve_metric(metric))
for metric in metrics]
results = []
# Build a set of operators
with multiprocessing.Pool(processes=len(operators)) as pool:
operator_exec_out = pool.starmap(run_operator,
[(operator, None, config, targets)
for operator in operators])
data = {}
for operator_name, result in operator_exec_out:
data[operator_name] = result
# Write a summary table
extra = []
for operator, metric in metrics:
if detail and resolve_operator(operator).level == OperatorLevel.Object:
for file in files:
try:
extra.extend([
f"{file}:{k}"
for k in data[operator][file]["detailed"].keys()
if k != metric.name and isinstance(
data[operator][file]["detailed"][k], dict)
])
except KeyError:
logger.debug(f"File {file} not in cache")
logger.debug("Cache follows -- ")
logger.debug(data[operator])
files.extend(extra)
logger.debug(files)
for file in files:
metrics_data = []
has_changes = False
for operator, metric in metrics:
try:
current = target_revision.get(config, state.default_archiver,
operator, file, metric.name)
except KeyError:
current = "-"
try:
new = get_metric(data, operator, file, metric.name)
except KeyError:
new = "-"
if new != current:
has_changes = True
if metric.type in (int, float) and new != "-" and current != "-":
if current > new:
metrics_data.append(
"{0:n} -> \u001b[{2}m{1:n}\u001b[0m".format(
current, new, BAD_COLORS[metric.measure]))
elif current < new:
metrics_data.append(
"{0:n} -> \u001b[{2}m{1:n}\u001b[0m".format(
current, new, GOOD_COLORS[metric.measure]))
else:
metrics_data.append("{0:n} -> {1:n}".format(current, new))
else:
if current == "-" and new == "-":
metrics_data.append("-")
else:
metrics_data.append("{0} -> {1}".format(current, new))
if has_changes or not changes_only:
results.append((file, *metrics_data))
else:
logger.debug(metrics_data)
descriptions = [metric.description for operator, metric in metrics]
headers = ("File", *descriptions)
if len(results) > 0:
print(
# But it still makes more sense to show the newest at the top, so reverse again
tabulate.tabulate(headers=headers,
tabular_data=results,
tablefmt=DEFAULT_GRID_STYLE))
def build(config, archiver, operators):
"""
Build the history given a archiver and collection of operators.
:param config: The wily configuration
:type config: :namedtuple:`wily.config.WilyConfig`
:param archiver: The archiver to use
:type archiver: :namedtuple:`wily.archivers.Archiver`
:param operators: The list of operators to execute
:type operators: `list` of :namedtuple:`wily.operators.Operator`
"""
try:
logger.debug(f"Using {archiver.name} archiver module")
archiver = archiver.cls(config)
revisions = archiver.revisions(config.path, config.max_revisions)
except InvalidGitRepositoryError:
# TODO: This logic shouldn't really be here (SoC)
logger.info(
f"Defaulting back to the filesystem archiver, not a valid git repo"
)
archiver = FilesystemArchiver(config)
revisions = archiver.revisions(config.path, config.max_revisions)
except Exception as e:
if hasattr(e, "message"):
logger.error(f"Failed to setup archiver: '{e.message}'")
else:
logger.error(f"Failed to setup archiver: '{type(e)} - {e}'")
exit(1)
state = State(config, archiver=archiver)
# Check for existence of cache, else provision
state.ensure_exists()
index = state.index[archiver.name]
# remove existing revisions from the list
revisions = [revision for revision in revisions if revision not in index]
logger.info(
f"Found {len(revisions)} revisions from '{archiver.name}' archiver in '{config.path}'."
)
_op_desc = ",".join([operator.name for operator in operators])
logger.info(f"Running operators - {_op_desc}")
bar = Bar("Processing", max=len(revisions) * len(operators))
state.operators = operators
try:
with multiprocessing.Pool(processes=len(operators)) as pool:
for revision in revisions:
# Checkout target revision
archiver.checkout(revision, config.checkout_options)
stats = {"operator_data": {}}
# Run each operator as a seperate process
data = pool.starmap(
run_operator,
[(operator, revision, config) for operator in operators],
)
# Map the data back into a dictionary
for operator_name, result in data:
# aggregate values to directories
roots = []
# find all unique directories in the results
for entry in result.keys():
parent = pathlib.Path(entry).parents[0]
if parent not in roots:
roots.append(parent)
for root in roots:
# find all matching entries recursively
aggregates = [
path for path in result.keys()
if root in pathlib.Path(path).parents
]
result[str(root)] = {"total": {}}
# aggregate values
for metric in resolve_operator(
operator_name).cls.metrics:
func = metric.aggregate
values = [
result[aggregate]["total"][metric.name]
for aggregate in aggregates
if aggregate in result
and metric.name in result[aggregate]["total"]
]
if len(values) > 0:
result[str(root)]["total"][metric.name] = func(
values)
stats["operator_data"][operator_name] = result
bar.next()
ir = index.add(revision, operators=operators)
ir.store(config, archiver, stats)
index.save()
bar.finish()
except Exception as e:
logger.error(f"Failed to build cache: '{e}'")
raise e
finally:
# Reset the archive after every run back to the head of the branch
archiver.finish()
