def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
data = req.args.get('payload')
branches = (parse_qs(req.query_string).get('branches') or self.branches).split(',')
self.env.log.debug("Using branches: %s", branches)
if data:
jsondata = simplejson.loads(data)
ref = jsondata['ref'].split('/')[-1]
if ref in branches or 'all' in branches:
for i in jsondata['commits']:
self.hook.process(i, status, jsondata)
else:
self.env.log.debug("Not running hook, ref %s is not in %s", ref, branches)
if self.autofetch:
repo = Git(self.repo)
try:
repo.execute(['git', 'fetch'])
except:
self.env.log.debug("git fetch failed!")
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
if self.autofetch:
repodir = RepositoryManager(self.env).repository_dir
if not os.path.isabs(repodir):
repodir = os.path.join(self.env.path, repodir)
# TODO: This was the previous code, the repo options is probably unecessary now.
# repodir = "%s/%s" % (self.repo, reponame)
self.env.log.debug("Autofetching: %s" % repodir)
repo = Git(repodir)
try:
self.env.log.debug("Fetching repo %s" % self.repo)
repo.execute(['git', 'fetch'])
try:
self.env.log.debug("Resyncing local repo")
self.env.get_repository('').sync()
except:
self.env.log.error("git sync failed!")
except:
self.env.log.error("git fetch failed!")
data = req.args.get('payload')
if data:
jsondata = simplejson.loads(data)
reponame = jsondata['repository']['name']
for i in jsondata['commits']:
self.hook.process(i, status, self.enable_revmap,reponame)
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
data = req.args.get('payload')
jsondata = simplejson.loads(data)
repoName = jsondata['repository']['name']
if self.autofetch:
if data:
jsondata = simplejson.loads(data)
self.env.log.debug(jsondata['repository']['name']);
repo = Git(self.gitreposdir+repoName+"/.git")
try:
self.env.log.debug("Fetching repo %s" % self.repo)
repo.execute(['git', 'fetch'])
try:
self.env.log.debug("Resyncing local repo")
self.env.get_repository(repoName).sync()
except:
self.env.log.error("git sync failed!")
except:
self.env.log.error("git fetch failed!")
jsondata = simplejson.loads(data)
if jsondata:
if jsondata['ref'] == "refs/heads/master" or re.search('-stable$', jsondata['ref']):
for i in jsondata['commits']:
self.hook.process(i, status)
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
data = req.args.get('payload')
branches = (self.parse_query_string(req.query_string).get('branches')
or self.branches).split(',')
self.env.log.debug("Using branches: %s", branches)
if data:
jsondata = simplejson.loads(data)
ref = jsondata['ref'].split('/')[-1]
if ref in branches or 'all' in branches:
for i in jsondata['commits']:
self.hook.process(i, status, jsondata)
else:
self.env.log.debug("Not running hook, ref %s is not in %s",
ref, branches)
if self.autofetch:
repo = Git(self.repo)
try:
repo.execute(['git', 'fetch'])
except:
self.env.log.debug("git fetch failed!")
def command(self, command):
"""
Runs the Git command in self.repo
"""
args = split(command)
cmd = Git(self.repodir)
cmd.execute(args)
def get_webservertime(host):
conn = http.client.HTTPConnection(host)
conn.request("GET", "/")
r = conn.getresponse()
ts = r.getheader('date')
r = Git("E:\\vstsworkspace\\projectXS")
r.execute('git commit --amend --no-verify --date="' + ts + '"')
r.execute("git log")
print("test")
def command(self, command):
"""
Runs the Git command in self.repo
"""
args = split(command)
cmd = Git(self.repodir)
cmd.execute(args)
class _GitWrapperCommon(object):
'''
Wrap git module to provide a more stable interface across versions
'''
def __init__(self, repo_path):
self.git = Git()
self.repo = Repo(os.path.abspath('.'))
def is_file_managed_by_git(self, path):
'''
:param path: Path to check
:returns: True if path is managed by git
'''
status, _stdout, _stderr = self.git.execute(
['git', 'ls-files', path, '--error-unmatch'],
with_extended_output=True,
with_exceptions=False)
return status == 0
def is_file_modified(self, path):
'''
Does a file have local changes not yet committed
:returns: True if file has local changes
'''
status, _stdout, _stderr = self.git.execute(
['git', 'diff', '--quiet', 'HEAD', path],
with_extended_output=True,
with_exceptions=False)
return status != 0
def get_commits_following(self, path):
'''
Get all commits including path following the file through
renames
:param path: Path which we will find commits for
:returns: Sequence of commit objects. Newest to oldest
'''
commit_shas = self.git.log('--pretty=%H', '--follow', '--',
path).splitlines()
return map(self.repo.commit, commit_shas)
def get_commits(self, path, follow=False):
'''
Get all commits including path
:param path: Path which we will find commits for
:param bool follow: If True we will follow path through renames
:returns: Sequence of commit objects. Newest to oldest
'''
if follow:
return self.get_commits_following(path)
else:
return self._get_commits(path)
class _GitWrapperCommon(object):
'''
Wrap git module to provide a more stable interface across versions
'''
def __init__(self, repo_path):
self.git = Git()
self.repo = Repo(os.path.abspath('.'))
def is_file_managed_by_git(self, path):
'''
:param path: Path to check
:returns: True if path is managed by git
'''
status, _stdout, _stderr = self.git.execute(
['git', 'ls-files', path, '--error-unmatch'],
with_extended_output=True,
with_exceptions=False)
return status == 0
def is_file_modified(self, path):
'''
Does a file have local changes not yet committed
:returns: True if file has local changes
'''
status, _stdout, _stderr = self.git.execute(
['git', 'diff', '--quiet', 'HEAD', path],
with_extended_output=True,
with_exceptions=False)
return status != 0
def get_commits_following(self, path):
'''
Get all commits including path following the file through
renames
:param path: Path which we will find commits for
:returns: Sequence of commit objects. Newest to oldest
'''
commit_shas = self.git.log(
'--pretty=%H', '--follow', '--', path).splitlines()
return map(self.repo.commit, commit_shas)
def get_commits(self, path, follow=False):
'''
Get all commits including path
:param path: Path which we will find commits for
:param bool follow: If True we will follow path through renames
:returns: Sequence of commit objects. Newest to oldest
'''
if follow:
return self.get_commits_following(path)
else:
return self._get_commits(path)
class _GitWrapperCommon(object):
"""
Wrap git module to provide a more stable interface across versions
"""
def __init__(self, repo_path):
self.git = Git()
self.repo = Repo(os.path.abspath("."))
def is_file_managed_by_git(self, path):
"""
:param path: Path to check
:returns: True if path is managed by git
"""
status, _stdout, _stderr = self.git.execute(
["git", "ls-files", path, "--error-unmatch"], with_extended_output=True, with_exceptions=False
)
return status == 0
def is_file_modified(self, path):
"""
Does a file have local changes not yet committed
:returns: True if file has local changes
"""
status, _stdout, _stderr = self.git.execute(
["git", "diff", "--quiet", "HEAD", path], with_extended_output=True, with_exceptions=False
)
return status != 0
def get_commits_following(self, path):
"""
Get all commits including path following the file through
renames
:param path: Path which we will find commits for
:returns: Sequence of commit objects. Newest to oldest
"""
commit_shas = self.git.log("--pretty=%H", "--follow", "--", path).splitlines()
return [self.repo.commit(shas) for shas in commit_shas]
def get_commits(self, path, follow=False):
"""
Get all commits including path
:param path: Path which we will find commits for
:param bool follow: If True we will follow path through renames
:returns: Sequence of commit objects. Newest to oldest
"""
if follow:
return self.get_commits_following(path)
else:
return self._get_commits(path)
def _get_changes(self, path, remotepath, gitclient=None):
''' Get and print changes from repository
TODO. Remove Git client and get it with naive clear
'''
check = Git(path)
if gitclient != None:
check = gitclient
check.execute(["git", "checkout", "master"])
try:
return self._merge(check)
except:
self._fetchUpstream(check, remotepath)
return self._merge(check)
class TestGit(object):
def setup(self):
base = os.path.join(os.path.dirname(__file__), "../..")
self.git = Git(base)
@patch(Git, 'execute')
def test_call_process_calls_execute(self, git):
git.return_value = ''
self.git.version()
assert_true(git.called)
assert_equal(git.call_args, ((['git', 'version'],), {}))
@raises(GitCommandError)
def test_it_raises_errors(self):
self.git.this_does_not_exist()
def test_it_transforms_kwargs_into_git_command_arguments(self):
assert_equal(["-s"], self.git.transform_kwargs(**{'s': True}))
assert_equal(["-s5"], self.git.transform_kwargs(**{'s': 5}))
assert_equal(["--max-count"], self.git.transform_kwargs(**{'max_count': True}))
assert_equal(["--max-count=5"], self.git.transform_kwargs(**{'max_count': 5}))
assert_equal(["-s", "-t"], self.git.transform_kwargs(**{'s': True, 't': True}))
def test_it_executes_git_to_shell_and_returns_result(self):
assert_match('^git version [\d\.]{2}.*$', self.git.execute(["git","version"]))
def test_it_accepts_stdin(self):
filename = fixture_path("cat_file_blob")
fh = open(filename, 'r')
assert_equal("70c379b63ffa0795fdbfbc128e5a2818397b7ef8",
self.git.hash_object(istream=fh, stdin=True))
fh.close()
def test_it_handles_large_input(self):
if sys.platform == 'win32':
output = self.git.execute(["type", "C:\WINDOWS\system32\cmd.exe"])
else:
output = self.git.execute(["cat", "/bin/bash"])
assert_true(len(output) > 4096) # at least 4k
@patch(Git, 'execute')
def test_it_ignores_false_kwargs(self, git):
# this_should_not_be_ignored=False implies it *should* be ignored
output = self.git.version(pass_this_kwarg=False)
assert_true("pass_this_kwarg" not in git.call_args[1])
def previous(rel_ver):
"""
Given a release version, find the previous version based on the latest Git
tag that is strictly a lower version than the given release version.
"""
if DEBUG:
print('Calculating previous release version (option -p was specified).')
version_loose = LooseVersion('0.0.0')
rel_ver_loose = LooseVersion(rel_ver)
gexc = Git('.')
tags = gexc.execute(['git', 'tag',
'--list', '1.*'])
for tag in tags.splitlines():
previous_tag_match = PREVIOUS_TAG_RE.match(tag)
if previous_tag_match:
version_new = {}
version_new['major'] = int(previous_tag_match.group('vermaj'))
version_new['minor'] = int(previous_tag_match.group('vermin'))
version_new['patch'] = int(previous_tag_match.group('verpatch'))
new_version_loose = LooseVersion(str(version_new['major']) + '.' +
str(version_new['minor']) + '.' +
str(version_new['patch']))
if new_version_loose < rel_ver_loose and new_version_loose > version_loose:
version_loose = new_version_loose
if DEBUG:
print('Found new best version "' + str(version_loose) \
+ '" from tag "' + tag + '"')
return str(version_loose)
def mgit(module_list=None, command=''):
"""
Work with multiple Git at the same time warehouse
"""
for module in module_list:
if not grant_mgit(module):
error_exit("this module have no grant from mgit!")
if not path_exist(module):
error_exit("this module don`t find from local disk!")
for module in module_list:
print(get_red_bold("---------------- %s ---------------- " % module))
try:
git = Git(__base_code_path__ % module)
except InvalidGitRepositoryError as e:
error_exit("InvalidGitRepositoryError: %s" % e)
continue
try:
output = git.execute(command=command, shell=True)
if output == '':
output = '%s execute done!' % command
print(get_color_text(output, foreground_color=ForegroundColor.GREEN))
except GitCommandError as e:
error_exit("GitCommandError: %s" % e)
continue
print('\n')
def merged_refs(self, skip=[]):
"""
Returns a list of remote refs that have been merged into the master
branch.
The "master" branch may have a different name than master. The value of
``self.master_name`` is used to determine what this name is.
"""
origin = self._origin
master = self._master_ref(origin)
refs = self._filtered_remotes(origin,
skip=['HEAD', self.master_branch] + skip)
merged = []
for ref in refs:
upstream = '{origin}/{master}'.format(origin=origin.name,
master=master.remote_head)
head = '{origin}/{branch}'.format(origin=origin.name,
branch=ref.remote_head)
cmd = Git(self.repo.working_dir)
# Drop to the git binary to do this, it's just easier to work with
# at this level.
(retcode, stdout,
stderr) = cmd.execute(['git', 'cherry', upstream, head],
with_extended_output=True,
with_exceptions=False)
if retcode == 0 and not stdout:
# This means there are no commits in the branch that are not
# also in the master branch. This is ready to be deleted.
merged.append(ref)
return merged
def merged_refs(self, skip=[]):
"""
Returns a list of remote refs that have been merged into the master
branch.
The "master" branch may have a different name than master. The value of
``self.master_name`` is used to determine what this name is.
"""
origin = self._origin
master = self._master_ref(origin)
refs = self._filtered_remotes(
origin, skip=['HEAD', self.master_branch] + skip)
merged = []
for ref in refs:
upstream = '{origin}/{master}'.format(
origin=origin.name, master=master.remote_head)
head = '{origin}/{branch}'.format(
origin=origin.name, branch=ref.remote_head)
cmd = Git(self.repo.working_dir)
# Drop to the git binary to do this, it's just easier to work with
# at this level.
(retcode, stdout, stderr) = cmd.execute(
['git', 'cherry', upstream, head],
with_extended_output=True, with_exceptions=False)
if retcode == 0 and not stdout:
# This means there are no commits in the branch that are not
# also in the master branch. This is ready to be deleted.
merged.append(ref)
return merged
def full_remote(self):
# alextodo. wrap the calls to git commit
repo = Git(self.path)
cmd = ['git', 'remote', '-v']
remote = repo.execute(cmd).split('(fetch)')[0]
remote = remote or ''
return remote.strip()
def previous(rel_ver):
"""
Given a release version, find the previous version based on the latest Git
tag that is strictly a lower version than the given release version.
"""
if DEBUG:
print 'Calculating previous release version (option -p was specified).'
version_loose = LooseVersion('0.0.0')
rel_ver_loose = LooseVersion(rel_ver)
gexc = Git('.')
tags = gexc.execute(['git', 'tag',
'--list', '1.*',
'--sort', '-version:refname'])
for tag in tags.splitlines():
previous_tag_match = PREVIOUS_TAG_RE.match(tag)
if previous_tag_match:
version_new = {}
version_new['major'] = int(previous_tag_match.group('vermaj'))
version_new['minor'] = int(previous_tag_match.group('vermin'))
version_new['patch'] = int(previous_tag_match.group('verpatch'))
new_version_loose = LooseVersion(str(version_new['major']) + '.' +
str(version_new['minor']) + '.' +
str(version_new['patch']))
if new_version_loose < rel_ver_loose:
version_loose = new_version_loose
if DEBUG:
print 'Found new best version "' + str(version_loose) \
+ '" from tag "' + tag + '"'
return str(version_loose)
return str(version_loose)
def full_remote(self):
# alextodo. wrap the calls to git commit
repo = Git(self.path)
cmd = ['git', 'remote', '-v']
remote = repo.execute(cmd).split('(fetch)')[0]
remote = remote or ''
return remote.strip()
def git_updates(self):
update_win = UpdateForm(self)
from git import Repo, Git
repo = Repo(civiltools_path)
tags = repo.git.tag(l=True).split('\n')
update_win.tag_list.addItems(tags)
if update_win.exec_():
tag = update_win.tag_list.currentItem().text()
else:
return
if tag != 'Latest':
g = Git(civiltools_path)
result = g.execute(['git', 'checkout', '-f', tag])
msg = f'You have successfully move to {tag}'
QtWidgets.QMessageBox.information(None, 'update', str(msg))
return
if (QMessageBox.question(self, "update", ("update to latest version?!"),
QMessageBox.Yes | QMessageBox.No) == QMessageBox.No):
return
if not internet():
msg = "You are not connected to the Internet, please check your internet connection."
QtWidgets.QMessageBox.warning(None, 'update', str(msg))
return
import git
g = git.cmd.Git(civiltools_path)
msg = ''
try:
msg = g.pull(env={'GIT_SSL_NO_VERIFY': '1'})
except:
QMessageBox.information(self, "update", "update takes some minutes, please be patient.")
import shutil
import tempfile
pkgs_dir = os.path.abspath(os.path.join(civiltools_path, os.path.pardir))
default_tmp_dir = tempfile._get_default_tempdir()
name = next(tempfile._get_candidate_names())
civiltools_temp_dir = os.path.join(default_tmp_dir, 'civiltools' + name)
os.mkdir(civiltools_temp_dir)
os.chdir(civiltools_temp_dir)
git.Git('.').clone("https://github.com/ebrahimraeyat/civilTools.git", env={'GIT_SSL_NO_VERIFY': '1'})
shutil.rmtree(civiltools_path, onerror=onerror)
src_folder = os.path.join(civiltools_temp_dir, 'civilTools')
shutil.copytree(src_folder, civiltools_path)
os.chdir(civiltools_path)
msg = 'update done successfully.'
# os.chdir(civiltools_path + '/..')
# pip_install = f'pip install --upgrade --install-option="--prefix={civiltools_path}/.." git+https://github.com/ebrahimraeyat/civilTools.git'
# subprocess.Popen([python_exe, '-m', pip_install])
else:
if not msg:
msg = 'error occured during update\nplease contact with @roknabadi'
# msg += '\n please restart the programm.'
QtWidgets.QMessageBox.information(None, 'update', msg)
def describe(self):
repo = Git(self.path)
cmd = ['git', 'describe', '--tags']
result = repo.execute(cmd).split('-')
if (len(result) == 1):
return '', 0, ''
else:
howmany, sha = result[-2:]
branch = '-'.join(result[0:len(result) - 2])
return branch, howmany, sha
def describe(self):
repo = Git(self.path)
cmd = ['git', 'describe', '--tags']
result = repo.execute(cmd).split('-')
if (len(result) == 1):
return '', 0, ''
else:
howmany, sha = result[-2:]
branch = '-'.join(result[0:len(result) - 2])
return branch, howmany, sha
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
data = req.args.get('payload')
if data:
jsondata = simplejson.loads(data)
for i in jsondata['commits']:
self.hook.process(i, status)
if self.autofetch:
repo = Git(self.repo)
try:
repo.execute(['git', 'fetch'])
except:
self.env.log.debug("git fetch failed!")
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
data = req.args.get('payload')
if data:
jsondata = simplejson.loads(data)
for i in jsondata['commits']:
self.hook.process(i, status)
if self.autofetch:
repo = Git(self.repo)
try:
repo.execute(['git', 'fetch'])
except:
self.env.log.debug("git fetch failed!")
def git_update(self):
import git
from git import Repo, Git
g = git.cmd.Git(str(civiltools_path))
msg = ''
try:
g.execute('git submodule update --init --recursive')
msg = g.execute(
f'git pull --recurse-submodules origin {self.branch}')
if not 'already' in msg.lower():
msg = 'update done successfully.'
except:
import shutil
import tempfile
default_tmp_dir = tempfile._get_default_tempdir()
name = next(tempfile._get_candidate_names())
civiltools_temp = Path(default_tmp_dir) / 'civiltools'
if not civiltools_temp.exists():
civiltools_temp.mkdir()
civiltools_temp_dir = Path(default_tmp_dir) / 'civiltools' / name
civiltools_temp_dir.mkdir()
os.chdir(str(civiltools_temp_dir))
g.execute(
f'git clone --branch {self.branch} --depth 1 --recurse-submodules {self.git_url}'
)
shutil.rmtree(str(civiltools_path), onerror=onerror)
src_folder = civiltools_temp_dir / 'civilTools'
shutil.copytree(str(src_folder), str(civiltools_path))
os.chdir(str(civiltools_path))
msg = 'update done successfully.'
else:
if not msg:
msg = 'error occured during update\nplease contact with @roknabadi\
Email: [email protected], [email protected]'
g = Git(civiltools_path)
g.execute(['git', 'checkout', self.branch])
return msg
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = "closed"
data = req.args.get("payload")
if data:
jsondata = simplejson.loads(data)
branch = jsondata["ref"][11:]
# if (branch == 'master') or branch.startswith('fixes/'):
for i in jsondata["commits"]:
self.hook.process(i, status, branch)
if self.autofetch:
repo = Git(self.repo)
try:
repo.execute(["git", "fetch"])
except:
self.env.log.debug("git fetch failed!")
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
data = req.args.get('payload')
if data:
jsondata = simplejson.loads(data)
reponame = jsondata['repository']['name']
for i in jsondata['commits']:
self.hook.process(i, status, reponame)
if self.autofetch:
repodir = "%s/%s" % (self.repo, reponame)
self.env.log.debug("Autofetching: %s" % repodir)
repo = Git(repodir)
try:
repo.execute(['git', 'fetch'])
except:
self.env.log.debug("git fetch failed!")
def processCommitHook(self, req):
self.env.log.debug("processCommitHook")
status = self.closestatus
if not status:
status = 'closed'
data = req.args.get('payload')
if data:
jsondata = simplejson.loads(data)
for i in jsondata['commits']:
self.hook.process(i, status, self.enable_revmap)
if int(self.autofetch):
repo = Git(self.repo)
try:
repo.execute(['git', 'fetch'])
except:
self.env.log.debug("git fetch failed!")
self.env.log.debug("Redirect URL: %s" % req)
req.redirect(self.browser)
def git_all_files():
print(green('** Retrieving git HEAD **'))
git = Git(REPOSITORY)
repo = Repo(REPOSITORY)
assert(not repo.bare)
head_commit = repo.head.commit
print(green('Head commit revision: %s' % head_commit))
print(green('message: %s' % head_commit.message))
result = git.execute(
['git', 'ls-tree', '--name-only', '-r', str(head_commit)]
)
files = result.split('\n')
return files
class GitHack(object):
def __init__(self, working_dir=r".."):
self.git = Git(working_dir)
# show executive command and result
@staticmethod
def show(command, res):
print(f"[ {command}]\n{res}")
# show info with date time and tips
@classmethod
def info(cls, tips=""):
print(f"{datetime.now()}: {tips}")
# show current active branch
def branch(self, para=""):
command = "git branch " + para
res = self.git.execute(command)
self.show(command, res)
# show status
def status(self, para=""):
command = "git status " + para
res = self.git.execute(command)
self.show(command, res)
def add(self, para="--all"):
command = "git add " + para
res = self.git.execute(command)
self.show(command, res)
def commit(self, para=" -m ", comment=""):
command = "git commit " + para + comment
res = self.git.execute(command)
self.show(command, res)
def push(self, para=""):
command = "git push " + para
res = self.git.execute(command)
self.show(command, res)
# show log
def log(self, para=""):
command = "git log " + para
res = self.git.execute(command)
self.show(command, res)
def checkout_to(repo_dir, commit_id, past_branch_name):
"""
@description :
switch current workspace to commit workspace
@param :
repo_dir:
commit_id:
@Returns :
repo:
"""
print('checkout workspace to {} from {}'.format(commit_id,
past_branch_name))
repo = Repo(repo_dir)
r = Git(repo_dir)
past_branch_name = past_branch_name
repo.create_head(past_branch_name, commit_id)
try:
r.execute('git checkout ' + past_branch_name + ' -f', shell=True)
except:
r.execute('git branch -D ' + past_branch_name, shell=True)
r.execute('git checkout ' + past_branch_name + ' -f', shell=True)
print('checkout end !')
return r
class _GitWrapperCommon(object):
"""
Wrap git module to provide a more stable interface across versions
"""
def __init__(self, repo_path):
self.git = Git()
self.git.update_environment(
GIT_CONFIG_NOSYSTEM="true", HOME=os.getcwd(), XDG_CONFIG_HOME=os.getcwd()
)
self.repo = Repo(os.path.abspath("."), search_parent_directories=True)
def is_file_managed_by_git(self, path):
"""
:param path: Path to check
:returns: True if path is managed by git
"""
status, _stdout, _stderr = self.git.execute(
["git", "ls-files", path, "--error-unmatch"],
with_extended_output=True,
with_exceptions=False,
)
return status == 0
def is_file_modified(self, path):
"""
Does a file have local changes not yet committed
:returns: True if file has local changes
"""
status, _stdout, _stderr = self.git.execute(
["git", "diff", "--quiet", "HEAD", path],
with_extended_output=True,
with_exceptions=False,
)
return status != 0
def get_commits_following(self, path):
"""
Get all commits including path following the file through
renames
:param path: Path which we will find commits for
:returns: Sequence of commit objects. Newest to oldest
"""
return [commit for commit, _ in self.get_commits_and_names_iter(path)]
def get_commits_and_names_iter(self, path):
"""
Get all commits including a given path following renames
"""
log_result = self.git.log(
"--pretty=%H", "--follow", "--name-only", "--", path
).splitlines()
for commit_sha, _, filename in grouper(log_result, 3):
yield self.repo.commit(commit_sha), filename
def get_commits(self, path, follow=False):
"""
Get all commits including path
:param path: Path which we will find commits for
:param bool follow: If True we will follow path through renames
:returns: Sequence of commit objects. Newest to oldest
"""
if follow:
return self.get_commits_following(path)
else:
return self._get_commits(path)
def parse_json(json_dic, dataset_dir, repo_dir):
s_type = 'Source Code'
status = 'Accepted'
print('start parse ...')
for project in json_dic:
# if not project == 'redis':
# continue
source_code_dir = os.path.join(dataset_dir, project, 'source-code')
if not os.path.exists(source_code_dir):
os.makedirs(source_code_dir)
product_info = json_dic[project]
info_list = product_info['info']
source_code = product_info['source_code']
if source_code == 'c':
source = 'c'
else:
source = 'cpp'
source_dir = os.path.join(repo_dir, source_code, project) # 工程源目录
target_dir = os.path.join(source_code_dir, project) # 工程现目录
#判断是否已经建立
# if os.path.exists(os.path.join(source_code_dir,'manifest.xml')):
# continue
# else:
if os.path.exists(target_dir):
# shutil.rmtree(target_dir)
pass
else:
print('start copy project files ...')
shutil.copytree(source_dir, target_dir)
print('copy successful ！！')
container = ET.Element('container')
repo = Repo(target_dir)
r = Git(target_dir)
for info in info_list:
#遍历每个commit
commit_id = info['commit_id']
cve_info = info['cve_info']
cwe_list = re.findall('cwe-[0-9]+', str(cve_info), re.I)
try:
for detail in RepositoryMining(
target_dir, single=commit_id).traverse_commits():
submissionDate = detail.committer_date.strftime('%Y-%m-%d')
author = detail.author
# 创建testcase 节点
testcase = ET.Element('testcase')
testcaseId = commit_id
testcase.set('id', testcaseId)
testcase.set('type', s_type)
testcase.set('status', status)
testcase.set('submissionDate', submissionDate)
testcase.set('language', source)
testcase.set('author', author.name)
# 创建testcase 节点
# 创建description节点
description = ET.Element('description')
description.text = detail.msg
testcase.append(description)
# 创建description节点
# 创建CVE节点
for cve_id in cve_info:
cve = ET.Element('CVE')
cve.set('id', cve_id)
testcase.append(cve)
#创建CVE节点
#创建CWE节点
for cwe in cwe_list:
cwe_ = ET.Element('CWE')
cwe_.set('id', cwe)
testcase.append(cwe_)
# 创建CWE节点
# 创建file节点
flaw_file_new_path = [
file.new_path.replace('\\', '/')
for file in detail.modifications if file.new_path
]
flaw_file_old_path = [
file.old_path.replace('\\', '/')
for file in detail.modifications if file.old_path
]
file_set = set(flaw_file_new_path + flaw_file_old_path)
#遍历与commit相关的文件
for file in detail.modifications:
diff_parsed = file.diff_parsed
del_lines = [i[0] for i in diff_parsed['deleted']]
add_lines = [i[0] for i in diff_parsed['added']]
if file.new_path:
new_path = file.new_path.replace('\\', '/')
# 创建new_file节点
new_file_node = ET.Element('file')
new_file_node.set('path', new_path)
new_file_node.set('language', source)
if len(del_lines) > 0:
for line in del_lines:
flaw = ET.Element('flaw')
flaw.set('type', str(1))
flaw.set('line', str(line))
new_file_node.append(flaw)
testcase.append(new_file_node)
# 创建new_file节点
if file.old_path:
old_path = file.old_path.replace('\\', '/')
# 创建old_file节点
old_file_node = ET.Element('file')
old_file_node.set('path', old_path)
old_file_node.set('language', source)
if len(add_lines) > 0:
for line in add_lines:
flaw = ET.Element('flaw')
flaw.set('type', str(0))
flaw.set('line', str(line))
old_file_node.append(flaw)
testcase.append(old_file_node)
# 创建old_file节点
# testcase 添加 file
#遍历与commit相关的文件
#遍历与commit不相关的文件
past_branch_name = project
repo.create_head(past_branch_name, commit_id)
try:
r.execute('git checkout ' + past_branch_name + ' -f',
shell=True)
except:
r.execute('git branch -D ' + past_branch_name,
shell=True)
r.execute('git checkout ' + past_branch_name + ' -f',
shell=True)
not_flaw_files = [
os.path.relpath(file, target_dir).replace('\\', '/')
for file in GitRepository(target_dir).files()
]
print(len(not_flaw_files), project)
try:
r.execute('git checkout master -f', shell=True)
except:
r.execute('git checkout unstable -f', shell=True)
r.execute('git branch -D ' + past_branch_name, shell=True)
# get the list of files present in the repo at the current commit
print('write file ....')
for file in not_flaw_files:
if file not in file_set:
if (
not file.endswith(".Cpp")
and not file.endswith(".c++")
and not file.endswith(".C")
and not file.endswith(".cxx")
and not file.endswith(".c")
and not file.endswith(".cpp")
and not file.endswith(".cc")
and not file.endswith(".h")
): # filter files that do not contain source code
continue
file_node = ET.Element('file')
file_node.set('path', file)
file_node.set('language', source)
testcase.append(file_node)
#遍历与commit不相关的文件
# container 添加 testcase
container.append(testcase)
except Exception as e:
# with open('log.txt','a+') as f:
# f.write(target_dir +'\n'+commit_id+'\n'+
# str(e)+'\n'+str(time.strftime('%Y-%m-%d %H:%M:%S'))+'\n')
print(e)
# 写入为manifest.xml
print('writing ' + project + ' manifest.xml')
prettyXml(container, '\t', '\n') # 执行美化方法
# ET.dump(root) # 显示出美化后的XML内容
tree = ET.ElementTree(container)
man_dir = os.path.join(source_code_dir, 'manifest.xml')
tree.write(man_dir, 'utf-8', True)
print('write successful!')
print('parse successful!')
# In[4]:
local = get_com_dict(r)
# local
# In[5]:
github = get_com_dict(r, origin=True)
# github
# In[6]:
github_dt = github[0]['Date']
local_dt = local[0]['Date']
# In[7]:
if github_dt > local_dt:
print('从github拉取')
r.execute('git pull', shell=True)
elif github_dt < local_dt:
print('同步到github')
r.execute('git push', shell=True)
else:
print('代码已经同步')
print('完成')
# In[ ]:
from git import Repo, Git
import io
r = Repo(path=".")
l = [
item for item in r.index.diff(None)
if item.a_path.endswith(".dat") and item.change_type == "M"
]
len(l)
for i in l:
print(i.a_path)
g = Git(Repo.git_dir)
with io.StringIO(
g.execute(["git", "--no-pager", "show",
f"HEAD~1:{i.a_path}"])) as old:
with open(i.a_path, mode="r+") as new:
with io.StringIO(new.read()) as copy:
new.seek(0)
line = old.readline()
while line and line.startswith("#"):
new.write(line)
line = old.readline()
line = copy.readline()
end = False
while line:
if not end and not line.startswith("#"):
end = True
if end:
new.write(line)
line = copy.readline()
new.truncate()
class _GitWrapperCommon(object):
'''
Wrap git module to provide a more stable interface across versions
'''
def __init__(self, repo_path):
self.git = Git()
self.git.update_environment(GIT_CONFIG_NOSYSTEM='true',
HOME=os.getcwd(),
XDG_CONFIG_HOME=os.getcwd())
self.repo = Repo(os.path.abspath('.'))
def is_file_managed_by_git(self, path):
'''
:param path: Path to check
:returns: True if path is managed by git
'''
status, _stdout, _stderr = self.git.execute(
['git', 'ls-files', path, '--error-unmatch'],
with_extended_output=True,
with_exceptions=False)
return status == 0
def is_file_modified(self, path):
'''
Does a file have local changes not yet committed
:returns: True if file has local changes
'''
status, _stdout, _stderr = self.git.execute(
['git', 'diff', '--quiet', 'HEAD', path],
with_extended_output=True,
with_exceptions=False)
return status != 0
def get_commits_following(self, path):
'''
Get all commits including path following the file through
renames
:param path: Path which we will find commits for
:returns: Sequence of commit objects. Newest to oldest
'''
return [commit for commit, _ in self.get_commits_and_names_iter(path)]
def get_commits_and_names_iter(self, path):
'''
Get all commits including a given path following renames
'''
log_result = self.git.log('--pretty=%H', '--follow', '--name-only',
'--', path).splitlines()
for commit_sha, _, filename in grouper(log_result, 3):
yield self.repo.commit(commit_sha), filename
def get_commits(self, path, follow=False):
'''
Get all commits including path
:param path: Path which we will find commits for
:param bool follow: If True we will follow path through renames
:returns: Sequence of commit objects. Newest to oldest
'''
if follow:
return self.get_commits_following(path)
else:
return self._get_commits(path)
#!/usr/bin/env python3
from git import Repo,Git
import io
r=Repo(path=".")
l=[item for item in r.index.diff("HEAD") if item.a_path.endswith(".dat") and item.change_type=="M"]
for i in l:
print(i.a_path)
g=Git(Repo.git_dir)
with io.StringIO(g.execute(["git" ,"--no-pager", "show", f"HEAD~1:{i.a_path}"])) as old:
with open(i.a_path,mode="r+") as new:
with io.StringIO(new.read()) as copy:
new.seek(0)
line=old.readline()
while line and line.startswith("#"):
new.write(line)
line=old.readline()
line=copy.readline()
end=False
while line:
if not end and not line.startswith("#"):
end=True
if end:
new.write(line)
line=copy.readline()
new.truncate()
pass
df = pd.read_csv('tomcat.csv', sep='|')
bug_ids = df['bug_id'].dropna().astype(int).tolist()
git = Git('C:\\Users\\aqd14\\Documents\\GitHub\\tomcat70')
# p = re.compile('commit \w+')
try:
with open('info.json', mode='w') as f:
d = dict()
for id in bug_ids:
output = git.execute(
command=
'git log --all --grep=https://bz.apache.org/bugzilla/show_bug.cgi?id={}'
.format(id))
print('id = {}'.format(id))
if not output:
print('This might not be a fix for bug id = {}'.format(id))
continue
# print(output)
commit_ids = extract_commit_ids(output)
# print(commit_ids)
revisions = extract_revisions(output)
# print(revisions)
class_info = get_class_changed(commit_ids, revisions)
d[id] = class_info
json.dump(d, fp=f, indent=4)
class GitService:
def __init__(self, work_dir):
self._work_dir = work_dir
self._git = Git(work_dir)
self._remote_url = None
self._name = None
self._domain = None
self._user = None
def init_from_remote(self):
self._remote_url = self._git.execute("git remote get-url origin")
matcher = re.match(r"git@(?P<domain>.+):(?P<user>.+)/(?P<name>.+).git", self._remote_url)
self._name = matcher.group("name")
self._domain = matcher.group("domain")
self._user = matcher.group("user")
def get_name(self):
return self._name
def get_ssh_url(self):
return self._remote_url
def get_http_url(self):
return f"https://{self._domain}/{self._user}/{self._name}"
def is_github(self):
return self._domain == "github.com"
def get_repo_id(self):
return f"{self._user}/{self._name}"
def init(self, ssh_url, auto_commit_all=False):
repo = Repo.init(self._work_dir)
repo.create_remote('origin', ssh_url)
if auto_commit_all:
self._git.execute('git add -A')
else:
readme = Path(self._work_dir + '/README.md')
if not readme.exists():
readme.touch()
repo.index.add('README.md')
repo.index.commit('init')
self._git.execute("git push --set-upstream origin master")
print("Created local repo")
def from_template(self, target_ssh_url, template_ssh_url='[email protected]:AlexanderWyss/Web-Starter.git'):
self._git.execute(f"git clone {template_ssh_url} .")
shutil.rmtree(self._work_dir + '/.git', ignore_errors=True)
self.init(target_ssh_url, True)
print("Created Repo from templated")
def commit(self, message):
self._git.execute('git add -A')
self._git.execute(f'git commit -m "{message}"')
self._git.execute('git push')
if('name' in node and (master or node['name']!='master')):
branches.append(node['name'])
return sorted(branches, reverse=True)
repos_data = getJsonData(JSON_URL, 'repos.json')
packet_list = getPackets(repos_data)
for packet_name in packet_list:
packet = getPacket(repos_data, packet_name)
if(packet==None):
continue
branches = getBranches(packet)
if(branches==None or len(branches)<1):
continue
git_url = getGitUrl(packet)
if(git_url == None):
continue
directory = (OUTPUT_DIRECTORY + packet_name)
checkDirectory(directory)
command = "git clone \'%s\' \'%s\' -b \'%s\'" % (git_url, directory, branches[0])
message = "\nGetting branch %s: \n\tfrom: %s\n\tto : %s\n\texec: %s" % (branches[0], git_url, directory, command)
print(message)
#call(['git', 'clone', git_url, directory, '-b', branches[0]])
g = Git( directory )
g.execute(['git', 'clone', git_url, directory, '-b', branches[0]])
def main():
"""
The algorithm is roughly:
1. Is the current HEAD associated with a tag that looks like a release
version?
2. If "yes" then use that as the version
3. If "no" then is the current branch master?
4. If "yes" the current branch is master, then inspect the branches that
fit the convention for a release branch and choose the latest;
increment the minor version, append .0 to form the new version (e.g.,
releases/v3.3 becomes 3.4.0), and append a pre-release marker
5. If "no" the current branch is not master, then determine the most
recent tag in history; strip any pre-release marker, increment the
patch version, and append a new pre-release marker
"""
repo = Repo('.')
assert not repo.bare
head_tag_ver = check_head_tag(repo)
if head_tag_ver:
return head_tag_ver
version_loose = LooseVersion('0.0.0')
prerelease_marker = datetime.date.today().strftime('%Y%m%d') \
+ '+git' + repo.head.commit.hexsha[:10]
if DEBUG:
print 'Calculating release version for branch: ' + repo.active_branch.name
if repo.active_branch.name == 'master':
version_new = {}
# Use refs (not branches) to get local branches plus remote branches
for ref in repo.refs:
release_branch_match = RELEASE_BRANCH_RE.match(ref.name)
if release_branch_match:
# Construct a candidate version from this branch name
version_new['major'] = int(release_branch_match.group('vermaj'))
version_new['minor'] = int(release_branch_match.group('vermin')) + 1
version_new['patch'] = 0
version_new['prerelease'] = prerelease_marker
new_version_loose = LooseVersion(str(version_new['major']) + '.' +
str(version_new['minor']) + '.' +
str(version_new['patch']) + '-' +
version_new['prerelease'])
if new_version_loose > version_loose:
version_loose = new_version_loose
if DEBUG:
print 'Found new best version "' + str(version_loose) \
+ '" on branch "' + ref.name + '"'
else:
gexc = Git('.')
tags = gexc.execute(['git', 'tag',
'--merged', 'HEAD',
'--list', 'r*',
'--sort', 'version:refname'])
if len(tags) > 0:
release_tag_match = RELEASE_TAG_RE.match(tags.splitlines()[-1])
if release_tag_match:
version_new = {}
version_new['major'] = int(release_tag_match.group('vermaj'))
version_new['minor'] = int(release_tag_match.group('vermin'))
version_new['patch'] = int(release_tag_match.group('verpatch')) + 1
version_new['prerelease'] = prerelease_marker
new_version_loose = LooseVersion(str(version_new['major']) + '.' +
str(version_new['minor']) + '.' +
str(version_new['patch']) + '-' +
version_new['prerelease'])
if new_version_loose > version_loose:
version_loose = new_version_loose
if DEBUG:
print 'Found new best version "' + str(version_loose) \
+ '" from tag "' + release_tag_match.group('ver') + '"'
return str(version_loose)
class _GitWrapperCommon(object):
'''
Wrap git module to provide a more stable interface across versions
'''
def __init__(self, repo_path):
self.git = Git()
self.git.update_environment(
GIT_CONFIG_NOSYSTEM='true',
HOME=os.getcwd(),
XDG_CONFIG_HOME=os.getcwd()
)
self.repo = Repo(os.path.abspath('.'))
def is_file_managed_by_git(self, path):
'''
:param path: Path to check
:returns: True if path is managed by git
'''
status, _stdout, _stderr = self.git.execute(
['git', 'ls-files', path, '--error-unmatch'],
with_extended_output=True,
with_exceptions=False)
return status == 0
def is_file_modified(self, path):
'''
Does a file have local changes not yet committed
:returns: True if file has local changes
'''
status, _stdout, _stderr = self.git.execute(
['git', 'diff', '--quiet', 'HEAD', path],
with_extended_output=True,
with_exceptions=False)
return status != 0
def get_commits_following(self, path):
'''
Get all commits including path following the file through
renames
:param path: Path which we will find commits for
:returns: Sequence of commit objects. Newest to oldest
'''
return [
commit for commit, _ in self.get_commits_and_names_iter(
path)]
def get_commits_and_names_iter(self, path):
'''
Get all commits including a given path following renames
'''
log_result = self.git.log(
'--pretty=%H',
'--follow',
'--name-only',
'--',
path).splitlines()
for commit_sha, _, filename in grouper(log_result, 3):
yield self.repo.commit(commit_sha), filename
def get_commits(self, path, follow=False):
'''
Get all commits including path
:param path: Path which we will find commits for
:param bool follow: If True we will follow path through renames
:returns: Sequence of commit objects. Newest to oldest
'''
if follow:
return self.get_commits_following(path)
else:
return self._get_commits(path)
