def get_text(self, sha=None):
if self.chapter.tutorial:
tutorial = self.chapter.tutorial
else:
tutorial = self.chapter.part.tutorial
repo = Repo(tutorial.get_path())
# find hash code
if sha is None:
sha = tutorial.sha_draft
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "parts" in tutorial_version:
for part in tutorial_version["parts"]:
if "chapters" in part:
for chapter in part["chapters"]:
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if "chapter" in tutorial_version:
chapter = tutorial_version["chapter"]
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if path_ext:
return get_blob(repo.commit(sha).tree, path_ext)
else:
return None
def get_text(self, sha=None):
if self.chapter.tutorial:
tutorial = self.chapter.tutorial
else:
tutorial = self.chapter.part.tutorial
repo = Repo(tutorial.get_path())
# find hash code
if sha is None:
sha = tutorial.sha_draft
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "parts" in tutorial_version:
for part in tutorial_version["parts"]:
if "chapters" in part:
for chapter in part["chapters"]:
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if "chapter" in tutorial_version:
chapter = tutorial_version["chapter"]
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if path_ext:
return get_blob(repo.commit(sha).tree, path_ext)
else:
return None
def get_conclusion(self, sha=None):
# find hash code
if sha is None:
sha = self.sha_draft
repo = Repo(self.get_path())
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "introduction" in tutorial_version:
path_tuto = tutorial_version["conclusion"]
if path_tuto:
return get_blob(repo.commit(sha).tree, path_tuto)
def get_conclusion(self, sha=None):
# find hash code
if sha is None:
sha = self.sha_draft
repo = Repo(self.get_path())
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "introduction" in tutorial_version:
path_tuto = tutorial_version["conclusion"]
if path_tuto:
return get_blob(repo.commit(sha).tree, path_tuto)
def load_json_for_public(self, sha=None):
if sha is None:
sha = self.sha_public
repo = Repo(self.get_path())
mantuto = get_blob(repo.commit(sha).tree, 'manifest.json')
data = json_reader.loads(mantuto)
return data
def load_json_for_public(self, sha=None):
if sha is None:
sha = self.sha_public
repo = Repo(self.get_path())
mantuto = get_blob(repo.commit(sha).tree, 'manifest.json')
data = json_reader.loads(mantuto)
return data
def load_json_for_public(self, sha=None):
if sha is None:
sha = self.sha_public
repo = Repo(self.get_path())
mantuto = get_blob(repo.commit(sha).tree, 'manifest.json')
data = json_reader.loads(mantuto)
if 'licence' in data:
data['licence'] = Licence.objects.filter(
code=data['licence']).first()
return data
def get_conclusion(self, sha=None):
tutorial = self.tutorial
# find hash code
if sha is None:
sha = tutorial.sha_draft
repo = Repo(tutorial.get_path())
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "parts" in tutorial_version:
for part in tutorial_version["parts"]:
if part["pk"] == self.pk:
path_part = part["conclusion"]
break
if path_part:
return get_blob(repo.commit(sha).tree, path_part)
else:
return None
def get_conclusion(self, sha=None):
tutorial = self.tutorial
# find hash code
if sha is None:
sha = tutorial.sha_draft
repo = Repo(tutorial.get_path())
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "parts" in tutorial_version:
for part in tutorial_version["parts"]:
if part["pk"] == self.pk:
path_part = part["conclusion"]
break
if path_part:
return get_blob(repo.commit(sha).tree, path_part)
else:
return None
def detect_from_commits_list(self, args):
historage = Repo(args.historage_dir)
extract_method_information = []
try:
for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
a_commit = historage.commit(a_commit_hash)
b_commit = historage.commit(b_commit_hash)
extract_method_information.extend(detect_extract_method_from_commit(a_commit, b_commit))
except ValueError:
print "Invalid input."
return
except BadObject, name:
print "Invalid hash of the commit:", name.message
def detect_from_commits_list(self, args):
historage = Repo(args.historage_dir)
results = []
try:
for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
a_commit = historage.commit(a_commit_hash)
b_commit = historage.commit(b_commit_hash)
results.extend(detect_shingle_pullup_method(a_commit, b_commit))
except ValueError:
print("Invalid input.")
return
except BadObject, name:
print("Invalid hash of the commit:", name.message)
def detect_from_commits_list(self, args):
historage = Repo(args.historage_dir)
extract_method_information = []
try:
for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
a_commit = historage.commit(a_commit_hash)
b_commit = historage.commit(b_commit_hash)
extract_method_information.extend(detect_extract_method_from_commit(a_commit, b_commit))
except ValueError:
print("Invalid input.")
return
except BadObject as name:
print("Invalid hash of the commit:", name.message)
self.print_candidates(extract_method_information, args.format)
def detect_from_commits_list(self, args):
historage = Repo(args.historage_dir)
results = []
try:
for a_commit_hash, b_commit_hash in csv.reader(
open(args.commits_list)):
a_commit = historage.commit(a_commit_hash)
b_commit = historage.commit(b_commit_hash)
results.extend(detect_shingle_pullup_method(
a_commit, b_commit))
except ValueError:
print("Invalid input.")
return
except BadObject as name:
print(("Invalid hash of the commit:", name.message))
for result in results:
# result[6] = sim
result[6] = str(result[6])
print((','.join(['"' + s + '"' for s in result])))
def load_json_for_public(self):
repo = Repo(self.get_path())
mantuto = get_blob(repo.commit(self.sha_public).tree, 'manifest.json')
data = json_reader.loads(mantuto)
return data
class GitArchiver(BaseArchiver):
"""Gitpython implementation of the base archiver."""
name = "git"
def __init__(self, config):
"""
Instantiate a new Git Archiver.
:param config: The wily configuration
:type config: :class:`wily.config.WilyConfig`
"""
try:
self.repo = Repo(config.path)
except git.exc.InvalidGitRepositoryError as e:
raise InvalidGitRepositoryError from e
self.config = config
if self.repo.head.is_detached:
self.current_branch = self.repo.head.object.hexsha
else:
self.current_branch = self.repo.active_branch
assert not self.repo.bare, "Not a Git repository"
def revisions(self, path: str, max_revisions: int) -> List[Revision]:
"""
Get the list of revisions.
:param path: the path to target.
:type path: ``str``
:param max_revisions: the maximum number of revisions.
:type max_revisions: ``int``
:return: A list of revisions.
:rtype: ``list`` of :class:`Revision`
"""
if self.repo.is_dirty():
raise DirtyGitRepositoryError(self.repo.untracked_files)
revisions = []
for commit in self.repo.iter_commits(self.current_branch,
max_count=max_revisions,
reverse=True):
tracked_files, tracked_dirs = get_tracked_files_dirs(
self.repo, commit)
if not commit.parents or not revisions:
added_files = tracked_files
modified_files = []
deleted_files = []
else:
added_files, modified_files, deleted_files = whatchanged(
commit, self.repo.commit(commit.hexsha + "~1"))
logger.debug(
f"For revision {commit.name_rev.split(' ')[0]} found:")
logger.debug(f"Tracked files: {tracked_files}")
logger.debug(f"Tracked directories: {tracked_dirs}")
logger.debug(f"Added files: {added_files}")
logger.debug(f"Modified files: {modified_files}")
logger.debug(f"Deleted files: {deleted_files}")
rev = Revision(
key=commit.name_rev.split(" ")[0],
author_name=commit.author.name,
author_email=commit.author.email,
date=commit.committed_date,
message=commit.message,
tracked_files=tracked_files,
tracked_dirs=tracked_dirs,
added_files=added_files,
modified_files=modified_files,
deleted_files=deleted_files,
)
revisions.append(rev)
return revisions[::-1]
def checkout(self, revision: Revision, options: Dict):
"""
Checkout a specific revision.
:param revision: The revision identifier.
:type revision: :class:`Revision`
:param options: Any additional options.
:type options: ``dict``
"""
rev = revision.key
self.repo.git.checkout(rev)
def finish(self):
"""
Clean up any state if processing completed/failed.
For git, will checkout HEAD on the original branch when finishing
"""
self.repo.git.checkout(self.current_branch)
self.repo.close()
def find(self, search: str) -> Revision:
"""
Search a string and return a single revision.
:param search: The search term.
:type search: ``str``
:return: An instance of revision.
:rtype: Instance of :class:`Revision`
"""
commit = self.repo.commit(search)
tracked_files, tracked_dirs = get_tracked_files_dirs(self.repo, commit)
if not commit.parents:
added_files = tracked_files
modified_files = []
deleted_files = []
else:
added_files, modified_files, deleted_files = whatchanged(
commit, self.repo.commit(commit.hexsha + "~1"))
return Revision(
key=commit.name_rev.split(" ")[0],
author_name=commit.author.name,
author_email=commit.author.email,
date=commit.committed_date,
message=commit.message,
tracked_files=tracked_files,
tracked_dirs=tracked_dirs,
added_files=added_files,
modified_files=modified_files,
deleted_files=deleted_files,
)
class HistorageConverter:
parser_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib', 'java-parser.jar')
def __init__(self, org_git_repo_dir, historage_dir, syntax_trees_dir=None):
if org_git_repo_dir:
self.org_repo = Repo(org_git_repo_dir)
self.check_and_make_working_dir(historage_dir)
self.historage_dir = historage_dir
self.use_tempdir = syntax_trees_dir is None
if self.use_tempdir:
self.syntax_trees_dir = mkdtemp()
print(self.syntax_trees_dir)
else:
self.check_and_make_working_dir(syntax_trees_dir)
self.syntax_trees_dir = syntax_trees_dir
self.num_commits = 0
self.is_bare_repo = False
def check_and_make_working_dir(self, path):
if os.path.isdir(path):
if os.listdir(path):
raise Exception('{0} is not an empty directory'.format(path))
else:
try:
os.mkdir(path)
except OSError:
print('Kenja cannot make a directory: {0}'.format(path))
raise
def is_target_blob(self, blob, ext):
return blob and blob.name.endswith(ext)
def parse_all_java_files(self):
print 'create paresr processes...'
parser_executor = ParserExecutor(self.syntax_trees_dir, self.parser_jar_path)
parsed_blob = set()
for commit in get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs):
self.num_commits = self.num_commits + 1
commit = self.org_repo.commit(commit)
if commit.parents:
for p in commit.parents:
for diff in p.diff(commit):
if self.is_target_blob(diff.b_blob, '.java'):
if diff.b_blob.hexsha not in parsed_blob:
parser_executor.parse_blob(diff.b_blob)
parsed_blob.add(diff.b_blob.hexsha)
else:
for entry in commit.tree.traverse():
if isinstance(entry, Blob) and self.is_target_blob(entry, '.java'):
if entry.hexsha not in parsed_blob:
parser_executor.parse_blob(entry)
parsed_blob.add(entry.hexsha)
print 'waiting parser processes'
parser_executor.join()
def prepare_base_repo(self):
base_repo = Repo.init(self.historage_dir, bare=self.is_bare_repo)
self.set_git_config(base_repo)
return base_repo
def set_git_config(self, repo):
reader = repo.config_reader() # global config
writer = repo.config_writer() # local config
user_key = 'user'
if not reader.has_option(user_key, 'name'):
if not writer.has_section(user_key):
writer.add_section(user_key)
writer.set(user_key, 'name', 'Kenja Converter')
if not reader.has_option(user_key, 'email'):
if not writer.has_section(user_key):
writer.add_section(user_key)
writer.set(user_key, 'email', '*****@*****.**')
def convert(self):
self.parse_all_java_files()
self.construct_historage()
def construct_historage(self):
print 'create historage...'
base_repo = self.prepare_base_repo()
committer = SyntaxTreesCommitter(Repo(self.org_repo.git_dir), base_repo, self.syntax_trees_dir)
num_commits = self.num_commits if self.num_commits != 0 else '???'
for num, commit in izip(count(), get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs)):
commit = self.org_repo.commit(commit)
print '[%d/%s] convert %s to: %s' % (num, num_commits, commit.hexsha, base_repo.git_dir)
committer.apply_change(commit)
committer.create_heads()
committer.create_tags()
if not self.is_bare_repo:
base_repo.head.reset(working_tree=True)
def __del__(self):
if self.use_tempdir and os.path.exists(self.syntax_trees_dir):
rmtree(self.syntax_trees_dir)
path = "jEdit"
url = "/Users/kenjif/msr_repos/git/jEdit"
write_submodule_config(f, name, path, url)
committed = {}
tags = {}
heads = {}
for tag_ref in repo.tags:
tags[tag_ref.commit.hexsha] = tag_ref.name
for head in repo.heads:
heads[head.commit.hexsha] = head.name
for commit_hexsha, num in izip(commits, count()):
print num, commit_hexsha
git = new_repo.git
commit = repo.commit(commit_hexsha)
parents = []
for parent in commit.parents:
parents.append(committed[parent.hexsha])
message = "[%s] from %s" % (num, commit_hexsha)
new_tree = create_submodule_tree(new_repo.odb, commit_hexsha)
new_commit = Commit.create_from_tree(new_repo, new_tree, message, parents)
if commit_hexsha in tags:
new_repo.create_tag(tags[commit_hexsha], ref=new_commit)
if commit_hexsha in heads:
new_repo.create_head(heads[commit_hexsha], commit=new_commit)
committed[commit_hexsha] = new_commit