def add_tags_to_database(connection, tags=None, git_repo=None, repo_url=None, verbose=True):
'''
Add tags to the database
Input:
connection (sqlite3.connection): the connection to the database
tags (list): a list of tags
git_repo (git_explorer.core.Git): to use for extracting the content
repo_url (str): if git_repo is not provided, a repository url is needed to initialize the git_repo
verbose (bool): "Definition of verbose: containing more words than necessary: WORDY"
'''
if git_repo == None and repo_url ==None:
raise ValueError('Provide a git_repo or a repo_url')
if git_repo == None:
git_repo = Git(repo_url, cache_path=GIT_CACHE)
git_repo.clone(skip_existing=False)
if repo_url==None:
repo_url = git_repo.get_url()
repo_url = re.sub('\.git$|/$', '', repo_url)
if tags == None:
tags = git_repo.get_tags()
elif type(tags) == str:
tags = [tags]
if len(tags) == 0:
return
cursor = connection.cursor()
# to not add duplicates
tags = list(dict.fromkeys(tags)) # to get only unique tags
cursor.execute("SELECT tag FROM tags WHERE repo_url = :repo_url AND tag IN {}".format(tuple(tags)), {'repo_url':repo_url})
tags_already_in_the_db = list(pd.read_sql("SELECT tag FROM tags WHERE tag IN {} and repo_url = '{}'".format(tuple(tags+[tags[0]]), repo_url), connection).tag)
tags_to_add = [tag for tag in tags if tag not in tags_already_in_the_db]
if len(tags_to_add) == 0:
cursor.close()
return
print(' Adding new tags to the database')
for tag in tqdm(tags_to_add):
try:
tag_timestamp = filter.get_timestamp_for_tag(tag, git_repo)
# add to database
cursor.execute("INSERT INTO tags VALUES (:tag, :repo_url, :tag_timestamp)",
{'tag':tag, 'repo_url':repo_url, 'tag_timestamp':str(tag_timestamp)})
except:
print(' Failed to add tag {}'.format(tag))
connection.commit()
if verbose: print(' {} / {} tags were already in the database and added the rest.'.format(len(tags_already_in_the_db), len(tags)))
cursor.close()
return
def add_commits_to_database(connection,
commit_ids,
git_repo=None,
repository_url=None,
driver=None,
with_message_references_content=False,
verbose=True):
'''
Add commits to the database
Input:
connection (sqlite3.connection): the connection to the database
commit_ids (list): a list of commit_ids
git_repo (git_explorer.core.Git): to use for extracting the content
repository_url (str): if git_repo is not provided, a repository url is needed to initialize the git_repo
driver: a webdriver can be provided to avoid javascript required pages
with_message_references_content (bool): to add commits references (requires additional time)
verbose (bool): "Definition of verbose: containing more words than necessary: WORDY"
'''
if git_repo == None and repository_url == None:
raise ValueError('Provide a git_repo or a repository_url')
if git_repo == None:
git_repo = Git(repository_url, cache_path=GIT_CACHE)
git_repo.clone(skip_existing=True)
if repository_url == None:
repository_url = git_repo.get_url()
repository_url = re.sub('\.git$|/$', '', repository_url)
if type(commit_ids) == str:
commit_ids = [commit_ids]
if len(commit_ids) == 0:
print('No commit IDs were provided')
return
cursor = connection.cursor()
# to not add duplicates
commit_ids = list(dict.fromkeys(commit_ids)) # to get only unique ids
commits_already_in_the_db = list(
pd.read_sql(
"SELECT id FROM commits WHERE id IN {} and repository_url = '{}'".
format(tuple(commit_ids + [commit_ids[0]]),
repository_url), connection).id)
commits_to_add = [
commit_id for commit_id in commit_ids
if commit_id not in commits_already_in_the_db
]
if len(commits_to_add) == 0:
cursor.close()
return
if verbose:
print(' {} / {} are already in the database, now adding the rest.'.
format(len(commits_already_in_the_db), len(commit_ids)))
for commit_id in tqdm(commits_to_add):
try:
# initialize commit object
commit = Commit(git_repo, commit_id)
# message execution is combined with timestamp execution to speed up to process
message = commit._exec.run(
['git', 'log', '--format=%B%n%ct', '-n1', commit._id])
timestamp = message.pop(-1)
diff = commit._exec.run([
'git', 'diff', '--unified=1', commit._id + "^.." + commit._id
])
changed_files = get_changed_files_from_diff(diff)
hunks = get_hunks_from_diff(diff)
preprocessed_message = rank.simpler_filter_text(message)
preprocessed_diff = rank.simpler_filter_text(
re.sub(
'[^A-Za-z0-9]+', ' ', ' '.join(
rank.extract_relevant_lines_from_commit_diff(diff))))
preprocessed_changed_files = rank.simpler_filter_text(
changed_files)
if with_message_references_content:
commit_message_reference_content = extract_commit_message_reference_content(
message, repository_url, driver)
preprocessed_commit_message_reference_content = rank.extract_n_most_occurring_words(
commit_message_reference_content, n=20)
else:
commit_message_reference_content, preprocessed_commit_message_reference_content = None, None
# add to database
with connection:
cursor.execute(
"INSERT INTO commits VALUES (:repository_url, :id, :timestamp, :message, :changed_files, :diff, :hunks, :commit_message_reference_content, :preprocessed_message, :preprocessed_diff, :preprocessed_changed_files, :preprocessed_commit_message_reference_content)",
{
'repository_url':
repository_url,
'id':
commit_id,
'timestamp':
str(timestamp),
'message':
str(message),
'changed_files':
str(changed_files),
'diff':
str(diff),
'hunks':
str(hunks),
'commit_message_reference_content':
commit_message_reference_content,
'preprocessed_message':
preprocessed_message,
'preprocessed_diff':
preprocessed_diff,
'preprocessed_changed_files':
preprocessed_changed_files,
'preprocessed_commit_message_reference_content':
preprocessed_commit_message_reference_content
})
except:
print(' Failed to add commit {}'.format(commit_id))
if verbose: print(' All commits have been added to the database.')
cursor.close()
return