def main():
config = process_arguments()
repo = git.Repo(config.project)
commit = repo.commit(config.version)
diff = commit.diff(config.fix_hash, create_patch=True)
config.patch = util.load_patch(config.patch.read())
results = {config.version: run(config, diff)}
from runner import determine_vulnerability_status
config.additions_threshold = 0.5
config.deletions_threshold = 0.25
determine_vulnerability_status(config, results)
print(json.dumps(results, indent=4))
def run_git(config, detection_call):
"""
Run detection method with the provided patch for the provided repo on the provided versions (or all versions
if none provided).
:param config: Config object with all parameters
:param detection_call: Visitor callback method to evaluate each version in a form detection(config, version_diff)
:return: a results dictionary
:rtype: dict
"""
sha1_regex = re.compile('([a-f0-9]{40})')
full_path = os.path.abspath(config.project)
try:
repo = git.Repo(full_path)
except git.exc.InvalidGitRepositoryError:
error('project path is not a valid git repository', True)
try:
active_branch = repo.active_branch
except TypeError:
if len(repo.branches) == 1:
active_branch = repo.branches[0]
repo.git.checkout(active_branch, force=True)
else:
error('Repository is in a detached HEAD state ' +
'and could not determine default branch.', True)
if config.start_version:
start_version = pkg_resources.parse_version(config.start_version)
else:
start_version = pkg_resources.SetuptoolsLegacyVersion('0.0.0')
versions = []
if config.versions:
try:
if os.path.exists(config.versions):
with open(config.versions, 'r') as file:
versions = file.read().strip().split('\n')
else:
versions = config.versions.strip().split(',')
except TypeError:
versions = config.versions.strip().split(',')
else:
for tag in repo.tags:
if start_version <= pkg_resources.parse_version(tag.name):
versions.append(tag.name)
if not config.debug:
versions = tqdm.tqdm(versions)
version_results = {}
patch_text = config.patch.read()
patch = util.load_patch(patch_text)
match = sha1_regex.match(patch_text.split()[1])
if match:
sha = match.group(1)
else:
raise Exception('No commit hash found in patch')
if config.debug:
print('Starting from commit sha {}'.format(sha))
try:
for version in versions:
if version not in repo.tags:
raise ValueError('No such version "{}"'.format(version))
# Get the version commit and diff it with the fixed version (the patch hash)
# The "a" side is the evaluated version, the "b" side is the patch
# The diff shows what "a" needs to become "b"
# The diff is a instance of 'git.diff.DiffIndex'
# The object 'git.diff.DiffIndex' is an array of objects 'git.diff.Diff'
# The object 'git.diff.Diff' prints the diff, so I can parse it as a patch
commit = repo.commit(version)
version_diff = commit.diff(sha, create_patch=True)
diffs = []
for diff in patch:
result = resolver.resolve_path(
repo, sha, version, diff.header.new_path, config.debug
)
if config.debug:
print(result)
header = diff.header._replace(path=result[0], status=result[1])
adjusted_diff = diff._replace(header=header)
diffs.append(adjusted_diff)
config.patch = diffs
# Call visitor detection method on current version with parameters in the config dictionary
version_results[version] = detection_call(config, version_diff)
if config.debug:
print('Removing {0}'.format(version))
except KeyboardInterrupt:
pass
except AssertionError:
error('assertion failed!')
version_results = None
except Exception as e:
error(str(e))
version_results = None
finally:
print('\r', end='')
return version_results
def main():
config = process_arguments()
config.patch = util.load_patch(config.patch.read())
print(json.dumps(run(config), indent=4))
def run(config, version_diffs):
total_patch_additions = 0
total_patch_deletions = 0
detected_patch_additions = 0
detected_patch_deletions = 0
ratios = {}
confident = True
for patch_diff in config.patch:
new_source_path = patch_diff.header.path
old_source_path = patch_diff.header.old_path
full_path = new_source_path
if full_path in [d.b_path for d in version_diffs]:
if config.debug:
print('Found {0}'.format(full_path))
else:
full_path = old_source_path
if full_path in [d.b_path for d in version_diffs]:
if config.debug:
print('WARNING: Found file at old path: {0}'.format(
full_path))
if blacklisted(full_path) or not whitelisted(full_path):
continue
ratios[patch_diff.header.path] = {}
detected_file_deletions = 0
detected_file_additions = 0
patch_deletions, patch_additions, _, _, _ = split_changes(patch_diff)
found = False
for version_diff in version_diffs:
# b_path is the path that will become the fix_patch after applying the diff to the evaluated version
if full_path == version_diff.b_path:
found = True
# Need to format the diff as a string parseable by whatthepatch
stripped_changes = str(version_diff.diff).replace(
"b'", "", 1).replace("b\"", "", 1).replace("\\n", "\n")
version_patch = "--- {}\n+++ {}\n{}".format(
full_path, full_path, stripped_changes)
version_diff_patch = util.load_patch(version_patch)
version_deletions, version_additions, _, _, _ = split_changes(
version_diff_patch[0])
# Keep a search index so it always searches forward
search_index = 0
for deletion in patch_deletions:
for i in range(search_index, len(version_deletions)):
if deletion == version_deletions[i]:
search_index = i
detected_file_deletions += 1
break
# Keep a search index so it always searches forward
search_index = 0
for addition in patch_additions:
for i in range(search_index, len(version_additions)):
if addition == version_additions[i]:
search_index = i
detected_file_additions += 1
break
break
if not found:
if config.debug:
print('WARNING: File {0} does not exist'.format(full_path))
total_file_additions = len(patch_additions)
total_file_deletions = len(patch_deletions)
detected_patch_additions += detected_file_additions
detected_patch_deletions += detected_file_deletions
total_patch_additions += total_file_additions
total_patch_deletions += total_file_deletions
# Detected means things that are missing from the evaluated version.
# So the ratio comes from the difference from the total (original).
added_ratio = (
total_file_additions - detected_file_additions
) / total_file_additions if total_file_additions > 0 else None
deleted_ratio = (
total_file_deletions - detected_file_deletions
) / total_file_deletions if total_file_deletions > 0 else None
ratios[patch_diff.header.path]['additions'] = added_ratio
ratios[patch_diff.header.path]['deletions'] = deleted_ratio
ratios[patch_diff.header.path]['status'] = patch_diff.header.status
result = {
'overall': {
'additions': (total_patch_additions - detected_patch_additions) /
total_patch_additions if total_patch_additions > 0 else None,
'deletions': (total_patch_deletions - detected_patch_deletions) /
total_patch_deletions if total_patch_deletions > 0 else None,
'confident':
confident
},
'breakdown': ratios
}
return result
def run_git(config):
sha1_regex = re.compile('([a-f0-9]{40})')
full_path = os.path.abspath(config.project)
try:
repo = git.Repo(full_path)
except git.exc.InvalidGitRepositoryError:
error('project path is not a valid git repository', True)
try:
active_branch = repo.active_branch
except TypeError:
if len(repo.branches) == 1:
active_branch = repo.branches[0]
repo.git.checkout(active_branch)
else:
error(
'Repository is in a detatched HEAD state ' +
'and could not determine default branch.', True)
if config.start_version:
start_version = pkg_resources.parse_version(config.start_version)
else:
start_version = pkg_resources.SetuptoolsLegacyVersion('0.0.0')
if config.versions:
if os.path.exists(config.versions):
with open(config.versions, 'r') as file:
versions = file.read().strip().split('\n')
else:
versions = config.versions.strip().split(',')
else:
versions = []
for tag in repo.tags:
if start_version <= pkg_resources.parse_version(tag.name):
versions.append(tag.name)
if not config.debug:
versions = tqdm.tqdm(versions)
version_results = {}
patch_text = config.patch.read()
patch = util.load_patch(patch_text)
match = sha1_regex.match(patch_text.split()[1])
if match:
sha = match.group(1)
else:
raise Exception('No commit hash found in patch')
if config.debug:
print('Starting from commit sha {}'.format(sha))
try:
for version in versions:
if version in repo.branches:
repo.git.branch('-D', version)
if config.debug:
print('Checking out {0}'.format(version))
if version not in repo.tags:
raise ValueError('No such version "{}"'.format(version))
repo.git.reset('--hard')
repo.git.clean('-df')
repo.git.checkout(version)
diffs = []
for diff in patch:
result = resolver.resolve_path(repo, sha, version,
diff.header.new_path,
config.debug)
if config.debug:
print(result)
header = diff.header._replace(path=result[0], status=result[1])
adjusted_diff = diff._replace(header=header)
diffs.append(adjusted_diff)
config.patch = diffs
version_results[version] = detector.run(config)
repo.git.checkout(active_branch)
if config.debug:
print('Removing {0}'.format(version))
except KeyboardInterrupt:
pass
except AssertionError:
error('assertion failed!')
version_results = None
raise
except Exception as e:
error(str(e))
version_results = None
raise
finally:
print('\r', end='')
repo.git.reset('--hard')
repo.git.clean('-df')
repo.git.checkout(active_branch)
return version_results
def main():
config = process_arguments()
config.patch = util.load_patch(config.patch.read())
print(json.dumps(run(config), indent=4))
def run_git(config):
sha1_regex = re.compile('([a-f0-9]{40})')
full_path = os.path.abspath(config.project)
try:
repo = git.Repo(full_path)
except git.exc.InvalidGitRepositoryError:
error('project path is not a valid git repository', True)
try:
active_branch = repo.active_branch
except TypeError:
if len(repo.branches) == 1:
active_branch = repo.branches[0]
repo.git.checkout(active_branch)
else:
error('Repository is in a detatched HEAD state ' +
'and could not determine default branch.', True)
if config.start_version:
start_version = pkg_resources.parse_version(config.start_version)
else:
start_version = pkg_resources.SetuptoolsLegacyVersion('0.0.0')
if config.versions:
if os.path.exists(config.versions):
with open(config.versions, 'r') as file:
versions = file.read().strip().split('\n')
else:
versions = config.versions.strip().split(',')
else:
versions = []
for tag in repo.tags:
if start_version <= pkg_resources.parse_version(tag.name):
versions.append(tag.name)
if not config.debug:
versions = tqdm.tqdm(versions)
version_results = {}
patch_text = config.patch.read()
patch = util.load_patch(patch_text)
match = sha1_regex.match(patch_text.split()[1])
if match:
sha = match.group(1)
else:
raise Exception('No commit hash found in patch')
if config.debug:
print('Starting from commit sha {}'.format(sha))
try:
for version in versions:
if version in repo.branches:
repo.git.branch('-D', version)
if config.debug:
print('Checking out {0}'.format(version))
if version not in repo.tags:
raise ValueError('No such version "{}"'.format(version))
repo.git.reset('--hard')
repo.git.clean('-df')
repo.git.checkout(version)
diffs = []
for diff in patch:
result = resolver.resolve_path(
repo, sha, version, diff.header.new_path, config.debug
)
if config.debug:
print(result)
header = diff.header._replace(path=result[0], status=result[1])
adjusted_diff = diff._replace(header=header)
diffs.append(adjusted_diff)
config.patch = diffs
version_results[version] = detector.run(config)
repo.git.checkout(active_branch)
if config.debug:
print('Removing {0}'.format(version))
except KeyboardInterrupt:
pass
except AssertionError:
error('assertion failed!')
version_results = None
raise
except Exception as e:
error(str(e))
version_results = None
raise
finally:
print('\r', end='')
repo.git.reset('--hard')
repo.git.clean('-df')
repo.git.checkout(active_branch)
return version_results
