def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = 'closed' data = req.args.get('payload') branches = (parse_qs(req.query_string).get('branches') or self.branches).split(',') self.env.log.debug("Using branches: %s", branches) if data: jsondata = simplejson.loads(data) ref = jsondata['ref'].split('/')[-1] if ref in branches or 'all' in branches: for i in jsondata['commits']: self.hook.process(i, status, jsondata) else: self.env.log.debug("Not running hook, ref %s is not in %s", ref, branches) if self.autofetch: repo = Git(self.repo) try: repo.execute(['git', 'fetch']) except: self.env.log.debug("git fetch failed!")
def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = 'closed' if self.autofetch: repodir = RepositoryManager(self.env).repository_dir if not os.path.isabs(repodir): repodir = os.path.join(self.env.path, repodir) # TODO: This was the previous code, the repo options is probably unecessary now. # repodir = "%s/%s" % (self.repo, reponame) self.env.log.debug("Autofetching: %s" % repodir) repo = Git(repodir) try: self.env.log.debug("Fetching repo %s" % self.repo) repo.execute(['git', 'fetch']) try: self.env.log.debug("Resyncing local repo") self.env.get_repository('').sync() except: self.env.log.error("git sync failed!") except: self.env.log.error("git fetch failed!") data = req.args.get('payload') if data: jsondata = simplejson.loads(data) reponame = jsondata['repository']['name'] for i in jsondata['commits']: self.hook.process(i, status, self.enable_revmap,reponame)
def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = 'closed' data = req.args.get('payload') jsondata = simplejson.loads(data) repoName = jsondata['repository']['name'] if self.autofetch: if data: jsondata = simplejson.loads(data) self.env.log.debug(jsondata['repository']['name']); repo = Git(self.gitreposdir+repoName+"/.git") try: self.env.log.debug("Fetching repo %s" % self.repo) repo.execute(['git', 'fetch']) try: self.env.log.debug("Resyncing local repo") self.env.get_repository(repoName).sync() except: self.env.log.error("git sync failed!") except: self.env.log.error("git fetch failed!") jsondata = simplejson.loads(data) if jsondata: if jsondata['ref'] == "refs/heads/master" or re.search('-stable$', jsondata['ref']): for i in jsondata['commits']: self.hook.process(i, status)
def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = 'closed' data = req.args.get('payload') branches = (self.parse_query_string(req.query_string).get('branches') or self.branches).split(',') self.env.log.debug("Using branches: %s", branches) if data: jsondata = simplejson.loads(data) ref = jsondata['ref'].split('/')[-1] if ref in branches or 'all' in branches: for i in jsondata['commits']: self.hook.process(i, status, jsondata) else: self.env.log.debug("Not running hook, ref %s is not in %s", ref, branches) if self.autofetch: repo = Git(self.repo) try: repo.execute(['git', 'fetch']) except: self.env.log.debug("git fetch failed!")
def command(self, command): """ Runs the Git command in self.repo """ args = split(command) cmd = Git(self.repodir) cmd.execute(args)
def get_webservertime(host): conn = http.client.HTTPConnection(host) conn.request("GET", "/") r = conn.getresponse() ts = r.getheader('date') r = Git("E:\\vstsworkspace\\projectXS") r.execute('git commit --amend --no-verify --date="' + ts + '"') r.execute("git log") print("test")
class _GitWrapperCommon(object): ''' Wrap git module to provide a more stable interface across versions ''' def __init__(self, repo_path): self.git = Git() self.repo = Repo(os.path.abspath('.')) def is_file_managed_by_git(self, path): ''' :param path: Path to check :returns: True if path is managed by git ''' status, _stdout, _stderr = self.git.execute( ['git', 'ls-files', path, '--error-unmatch'], with_extended_output=True, with_exceptions=False) return status == 0 def is_file_modified(self, path): ''' Does a file have local changes not yet committed :returns: True if file has local changes ''' status, _stdout, _stderr = self.git.execute( ['git', 'diff', '--quiet', 'HEAD', path], with_extended_output=True, with_exceptions=False) return status != 0 def get_commits_following(self, path): ''' Get all commits including path following the file through renames :param path: Path which we will find commits for :returns: Sequence of commit objects. Newest to oldest ''' commit_shas = self.git.log('--pretty=%H', '--follow', '--', path).splitlines() return map(self.repo.commit, commit_shas) def get_commits(self, path, follow=False): ''' Get all commits including path :param path: Path which we will find commits for :param bool follow: If True we will follow path through renames :returns: Sequence of commit objects. Newest to oldest ''' if follow: return self.get_commits_following(path) else: return self._get_commits(path)
class _GitWrapperCommon(object): ''' Wrap git module to provide a more stable interface across versions ''' def __init__(self, repo_path): self.git = Git() self.repo = Repo(os.path.abspath('.')) def is_file_managed_by_git(self, path): ''' :param path: Path to check :returns: True if path is managed by git ''' status, _stdout, _stderr = self.git.execute( ['git', 'ls-files', path, '--error-unmatch'], with_extended_output=True, with_exceptions=False) return status == 0 def is_file_modified(self, path): ''' Does a file have local changes not yet committed :returns: True if file has local changes ''' status, _stdout, _stderr = self.git.execute( ['git', 'diff', '--quiet', 'HEAD', path], with_extended_output=True, with_exceptions=False) return status != 0 def get_commits_following(self, path): ''' Get all commits including path following the file through renames :param path: Path which we will find commits for :returns: Sequence of commit objects. Newest to oldest ''' commit_shas = self.git.log( '--pretty=%H', '--follow', '--', path).splitlines() return map(self.repo.commit, commit_shas) def get_commits(self, path, follow=False): ''' Get all commits including path :param path: Path which we will find commits for :param bool follow: If True we will follow path through renames :returns: Sequence of commit objects. Newest to oldest ''' if follow: return self.get_commits_following(path) else: return self._get_commits(path)
class _GitWrapperCommon(object): """ Wrap git module to provide a more stable interface across versions """ def __init__(self, repo_path): self.git = Git() self.repo = Repo(os.path.abspath(".")) def is_file_managed_by_git(self, path): """ :param path: Path to check :returns: True if path is managed by git """ status, _stdout, _stderr = self.git.execute( ["git", "ls-files", path, "--error-unmatch"], with_extended_output=True, with_exceptions=False ) return status == 0 def is_file_modified(self, path): """ Does a file have local changes not yet committed :returns: True if file has local changes """ status, _stdout, _stderr = self.git.execute( ["git", "diff", "--quiet", "HEAD", path], with_extended_output=True, with_exceptions=False ) return status != 0 def get_commits_following(self, path): """ Get all commits including path following the file through renames :param path: Path which we will find commits for :returns: Sequence of commit objects. Newest to oldest """ commit_shas = self.git.log("--pretty=%H", "--follow", "--", path).splitlines() return [self.repo.commit(shas) for shas in commit_shas] def get_commits(self, path, follow=False): """ Get all commits including path :param path: Path which we will find commits for :param bool follow: If True we will follow path through renames :returns: Sequence of commit objects. Newest to oldest """ if follow: return self.get_commits_following(path) else: return self._get_commits(path)
def _get_changes(self, path, remotepath, gitclient=None): ''' Get and print changes from repository TODO. Remove Git client and get it with naive clear ''' check = Git(path) if gitclient != None: check = gitclient check.execute(["git", "checkout", "master"]) try: return self._merge(check) except: self._fetchUpstream(check, remotepath) return self._merge(check)
class TestGit(object): def setup(self): base = os.path.join(os.path.dirname(__file__), "../..") self.git = Git(base) @patch(Git, 'execute') def test_call_process_calls_execute(self, git): git.return_value = '' self.git.version() assert_true(git.called) assert_equal(git.call_args, ((['git', 'version'],), {})) @raises(GitCommandError) def test_it_raises_errors(self): self.git.this_does_not_exist() def test_it_transforms_kwargs_into_git_command_arguments(self): assert_equal(["-s"], self.git.transform_kwargs(**{'s': True})) assert_equal(["-s5"], self.git.transform_kwargs(**{'s': 5})) assert_equal(["--max-count"], self.git.transform_kwargs(**{'max_count': True})) assert_equal(["--max-count=5"], self.git.transform_kwargs(**{'max_count': 5})) assert_equal(["-s", "-t"], self.git.transform_kwargs(**{'s': True, 't': True})) def test_it_executes_git_to_shell_and_returns_result(self): assert_match('^git version [\d\.]{2}.*$', self.git.execute(["git","version"])) def test_it_accepts_stdin(self): filename = fixture_path("cat_file_blob") fh = open(filename, 'r') assert_equal("70c379b63ffa0795fdbfbc128e5a2818397b7ef8", self.git.hash_object(istream=fh, stdin=True)) fh.close() def test_it_handles_large_input(self): if sys.platform == 'win32': output = self.git.execute(["type", "C:\WINDOWS\system32\cmd.exe"]) else: output = self.git.execute(["cat", "/bin/bash"]) assert_true(len(output) > 4096) # at least 4k @patch(Git, 'execute') def test_it_ignores_false_kwargs(self, git): # this_should_not_be_ignored=False implies it *should* be ignored output = self.git.version(pass_this_kwarg=False) assert_true("pass_this_kwarg" not in git.call_args[1])
def previous(rel_ver): """ Given a release version, find the previous version based on the latest Git tag that is strictly a lower version than the given release version. """ if DEBUG: print('Calculating previous release version (option -p was specified).') version_loose = LooseVersion('0.0.0') rel_ver_loose = LooseVersion(rel_ver) gexc = Git('.') tags = gexc.execute(['git', 'tag', '--list', '1.*']) for tag in tags.splitlines(): previous_tag_match = PREVIOUS_TAG_RE.match(tag) if previous_tag_match: version_new = {} version_new['major'] = int(previous_tag_match.group('vermaj')) version_new['minor'] = int(previous_tag_match.group('vermin')) version_new['patch'] = int(previous_tag_match.group('verpatch')) new_version_loose = LooseVersion(str(version_new['major']) + '.' + str(version_new['minor']) + '.' + str(version_new['patch'])) if new_version_loose < rel_ver_loose and new_version_loose > version_loose: version_loose = new_version_loose if DEBUG: print('Found new best version "' + str(version_loose) \ + '" from tag "' + tag + '"') return str(version_loose)
def mgit(module_list=None, command=''): """ Work with multiple Git at the same time warehouse """ for module in module_list: if not grant_mgit(module): error_exit("this module have no grant from mgit!") if not path_exist(module): error_exit("this module don`t find from local disk!") for module in module_list: print(get_red_bold("---------------- %s ---------------- " % module)) try: git = Git(__base_code_path__ % module) except InvalidGitRepositoryError as e: error_exit("InvalidGitRepositoryError: %s" % e) continue try: output = git.execute(command=command, shell=True) if output == '': output = '%s execute done!' % command print(get_color_text(output, foreground_color=ForegroundColor.GREEN)) except GitCommandError as e: error_exit("GitCommandError: %s" % e) continue print('\n')
def merged_refs(self, skip=[]): """ Returns a list of remote refs that have been merged into the master branch. The "master" branch may have a different name than master. The value of ``self.master_name`` is used to determine what this name is. """ origin = self._origin master = self._master_ref(origin) refs = self._filtered_remotes(origin, skip=['HEAD', self.master_branch] + skip) merged = [] for ref in refs: upstream = '{origin}/{master}'.format(origin=origin.name, master=master.remote_head) head = '{origin}/{branch}'.format(origin=origin.name, branch=ref.remote_head) cmd = Git(self.repo.working_dir) # Drop to the git binary to do this, it's just easier to work with # at this level. (retcode, stdout, stderr) = cmd.execute(['git', 'cherry', upstream, head], with_extended_output=True, with_exceptions=False) if retcode == 0 and not stdout: # This means there are no commits in the branch that are not # also in the master branch. This is ready to be deleted. merged.append(ref) return merged
def merged_refs(self, skip=[]): """ Returns a list of remote refs that have been merged into the master branch. The "master" branch may have a different name than master. The value of ``self.master_name`` is used to determine what this name is. """ origin = self._origin master = self._master_ref(origin) refs = self._filtered_remotes( origin, skip=['HEAD', self.master_branch] + skip) merged = [] for ref in refs: upstream = '{origin}/{master}'.format( origin=origin.name, master=master.remote_head) head = '{origin}/{branch}'.format( origin=origin.name, branch=ref.remote_head) cmd = Git(self.repo.working_dir) # Drop to the git binary to do this, it's just easier to work with # at this level. (retcode, stdout, stderr) = cmd.execute( ['git', 'cherry', upstream, head], with_extended_output=True, with_exceptions=False) if retcode == 0 and not stdout: # This means there are no commits in the branch that are not # also in the master branch. This is ready to be deleted. merged.append(ref) return merged
def full_remote(self): # alextodo. wrap the calls to git commit repo = Git(self.path) cmd = ['git', 'remote', '-v'] remote = repo.execute(cmd).split('(fetch)')[0] remote = remote or '' return remote.strip()
def previous(rel_ver): """ Given a release version, find the previous version based on the latest Git tag that is strictly a lower version than the given release version. """ if DEBUG: print 'Calculating previous release version (option -p was specified).' version_loose = LooseVersion('0.0.0') rel_ver_loose = LooseVersion(rel_ver) gexc = Git('.') tags = gexc.execute(['git', 'tag', '--list', '1.*', '--sort', '-version:refname']) for tag in tags.splitlines(): previous_tag_match = PREVIOUS_TAG_RE.match(tag) if previous_tag_match: version_new = {} version_new['major'] = int(previous_tag_match.group('vermaj')) version_new['minor'] = int(previous_tag_match.group('vermin')) version_new['patch'] = int(previous_tag_match.group('verpatch')) new_version_loose = LooseVersion(str(version_new['major']) + '.' + str(version_new['minor']) + '.' + str(version_new['patch'])) if new_version_loose < rel_ver_loose: version_loose = new_version_loose if DEBUG: print 'Found new best version "' + str(version_loose) \ + '" from tag "' + tag + '"' return str(version_loose) return str(version_loose)
def git_updates(self): update_win = UpdateForm(self) from git import Repo, Git repo = Repo(civiltools_path) tags = repo.git.tag(l=True).split('\n') update_win.tag_list.addItems(tags) if update_win.exec_(): tag = update_win.tag_list.currentItem().text() else: return if tag != 'Latest': g = Git(civiltools_path) result = g.execute(['git', 'checkout', '-f', tag]) msg = f'You have successfully move to {tag}' QtWidgets.QMessageBox.information(None, 'update', str(msg)) return if (QMessageBox.question(self, "update", ("update to latest version?!"), QMessageBox.Yes | QMessageBox.No) == QMessageBox.No): return if not internet(): msg = "You are not connected to the Internet, please check your internet connection." QtWidgets.QMessageBox.warning(None, 'update', str(msg)) return import git g = git.cmd.Git(civiltools_path) msg = '' try: msg = g.pull(env={'GIT_SSL_NO_VERIFY': '1'}) except: QMessageBox.information(self, "update", "update takes some minutes, please be patient.") import shutil import tempfile pkgs_dir = os.path.abspath(os.path.join(civiltools_path, os.path.pardir)) default_tmp_dir = tempfile._get_default_tempdir() name = next(tempfile._get_candidate_names()) civiltools_temp_dir = os.path.join(default_tmp_dir, 'civiltools' + name) os.mkdir(civiltools_temp_dir) os.chdir(civiltools_temp_dir) git.Git('.').clone("https://github.com/ebrahimraeyat/civilTools.git", env={'GIT_SSL_NO_VERIFY': '1'}) shutil.rmtree(civiltools_path, onerror=onerror) src_folder = os.path.join(civiltools_temp_dir, 'civilTools') shutil.copytree(src_folder, civiltools_path) os.chdir(civiltools_path) msg = 'update done successfully.' # os.chdir(civiltools_path + '/..') # pip_install = f'pip install --upgrade --install-option="--prefix={civiltools_path}/.." git+https://github.com/ebrahimraeyat/civilTools.git' # subprocess.Popen([python_exe, '-m', pip_install]) else: if not msg: msg = 'error occured during update\nplease contact with @roknabadi' # msg += '\n please restart the programm.' QtWidgets.QMessageBox.information(None, 'update', msg)
def describe(self): repo = Git(self.path) cmd = ['git', 'describe', '--tags'] result = repo.execute(cmd).split('-') if (len(result) == 1): return '', 0, '' else: howmany, sha = result[-2:] branch = '-'.join(result[0:len(result) - 2]) return branch, howmany, sha
def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = 'closed' data = req.args.get('payload') if data: jsondata = simplejson.loads(data) for i in jsondata['commits']: self.hook.process(i, status) if self.autofetch: repo = Git(self.repo) try: repo.execute(['git', 'fetch']) except: self.env.log.debug("git fetch failed!")
def git_update(self): import git from git import Repo, Git g = git.cmd.Git(str(civiltools_path)) msg = '' try: g.execute('git submodule update --init --recursive') msg = g.execute( f'git pull --recurse-submodules origin {self.branch}') if not 'already' in msg.lower(): msg = 'update done successfully.' except: import shutil import tempfile default_tmp_dir = tempfile._get_default_tempdir() name = next(tempfile._get_candidate_names()) civiltools_temp = Path(default_tmp_dir) / 'civiltools' if not civiltools_temp.exists(): civiltools_temp.mkdir() civiltools_temp_dir = Path(default_tmp_dir) / 'civiltools' / name civiltools_temp_dir.mkdir() os.chdir(str(civiltools_temp_dir)) g.execute( f'git clone --branch {self.branch} --depth 1 --recurse-submodules {self.git_url}' ) shutil.rmtree(str(civiltools_path), onerror=onerror) src_folder = civiltools_temp_dir / 'civilTools' shutil.copytree(str(src_folder), str(civiltools_path)) os.chdir(str(civiltools_path)) msg = 'update done successfully.' else: if not msg: msg = 'error occured during update\nplease contact with @roknabadi\ Email: [email protected], [email protected]' g = Git(civiltools_path) g.execute(['git', 'checkout', self.branch]) return msg
def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = "closed" data = req.args.get("payload") if data: jsondata = simplejson.loads(data) branch = jsondata["ref"][11:] # if (branch == 'master') or branch.startswith('fixes/'): for i in jsondata["commits"]: self.hook.process(i, status, branch) if self.autofetch: repo = Git(self.repo) try: repo.execute(["git", "fetch"]) except: self.env.log.debug("git fetch failed!")
def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = 'closed' data = req.args.get('payload') if data: jsondata = simplejson.loads(data) reponame = jsondata['repository']['name'] for i in jsondata['commits']: self.hook.process(i, status, reponame) if self.autofetch: repodir = "%s/%s" % (self.repo, reponame) self.env.log.debug("Autofetching: %s" % repodir) repo = Git(repodir) try: repo.execute(['git', 'fetch']) except: self.env.log.debug("git fetch failed!")
def processCommitHook(self, req): self.env.log.debug("processCommitHook") status = self.closestatus if not status: status = 'closed' data = req.args.get('payload') if data: jsondata = simplejson.loads(data) for i in jsondata['commits']: self.hook.process(i, status, self.enable_revmap) if int(self.autofetch): repo = Git(self.repo) try: repo.execute(['git', 'fetch']) except: self.env.log.debug("git fetch failed!") self.env.log.debug("Redirect URL: %s" % req) req.redirect(self.browser)
def git_all_files(): print(green('** Retrieving git HEAD **')) git = Git(REPOSITORY) repo = Repo(REPOSITORY) assert(not repo.bare) head_commit = repo.head.commit print(green('Head commit revision: %s' % head_commit)) print(green('message: %s' % head_commit.message)) result = git.execute( ['git', 'ls-tree', '--name-only', '-r', str(head_commit)] ) files = result.split('\n') return files
class GitHack(object): def __init__(self, working_dir=r".."): self.git = Git(working_dir) # show executive command and result @staticmethod def show(command, res): print(f"[ {command}]\n{res}") # show info with date time and tips @classmethod def info(cls, tips=""): print(f"{datetime.now()}: {tips}") # show current active branch def branch(self, para=""): command = "git branch " + para res = self.git.execute(command) self.show(command, res) # show status def status(self, para=""): command = "git status " + para res = self.git.execute(command) self.show(command, res) def add(self, para="--all"): command = "git add " + para res = self.git.execute(command) self.show(command, res) def commit(self, para=" -m ", comment=""): command = "git commit " + para + comment res = self.git.execute(command) self.show(command, res) def push(self, para=""): command = "git push " + para res = self.git.execute(command) self.show(command, res) # show log def log(self, para=""): command = "git log " + para res = self.git.execute(command) self.show(command, res)
def checkout_to(repo_dir, commit_id, past_branch_name): """ @description : switch current workspace to commit workspace @param : repo_dir: commit_id: @Returns : repo: """ print('checkout workspace to {} from {}'.format(commit_id, past_branch_name)) repo = Repo(repo_dir) r = Git(repo_dir) past_branch_name = past_branch_name repo.create_head(past_branch_name, commit_id) try: r.execute('git checkout ' + past_branch_name + ' -f', shell=True) except: r.execute('git branch -D ' + past_branch_name, shell=True) r.execute('git checkout ' + past_branch_name + ' -f', shell=True) print('checkout end !') return r
class _GitWrapperCommon(object): """ Wrap git module to provide a more stable interface across versions """ def __init__(self, repo_path): self.git = Git() self.git.update_environment( GIT_CONFIG_NOSYSTEM="true", HOME=os.getcwd(), XDG_CONFIG_HOME=os.getcwd() ) self.repo = Repo(os.path.abspath("."), search_parent_directories=True) def is_file_managed_by_git(self, path): """ :param path: Path to check :returns: True if path is managed by git """ status, _stdout, _stderr = self.git.execute( ["git", "ls-files", path, "--error-unmatch"], with_extended_output=True, with_exceptions=False, ) return status == 0 def is_file_modified(self, path): """ Does a file have local changes not yet committed :returns: True if file has local changes """ status, _stdout, _stderr = self.git.execute( ["git", "diff", "--quiet", "HEAD", path], with_extended_output=True, with_exceptions=False, ) return status != 0 def get_commits_following(self, path): """ Get all commits including path following the file through renames :param path: Path which we will find commits for :returns: Sequence of commit objects. Newest to oldest """ return [commit for commit, _ in self.get_commits_and_names_iter(path)] def get_commits_and_names_iter(self, path): """ Get all commits including a given path following renames """ log_result = self.git.log( "--pretty=%H", "--follow", "--name-only", "--", path ).splitlines() for commit_sha, _, filename in grouper(log_result, 3): yield self.repo.commit(commit_sha), filename def get_commits(self, path, follow=False): """ Get all commits including path :param path: Path which we will find commits for :param bool follow: If True we will follow path through renames :returns: Sequence of commit objects. Newest to oldest """ if follow: return self.get_commits_following(path) else: return self._get_commits(path)
def parse_json(json_dic, dataset_dir, repo_dir): s_type = 'Source Code' status = 'Accepted' print('start parse ...') for project in json_dic: # if not project == 'redis': # continue source_code_dir = os.path.join(dataset_dir, project, 'source-code') if not os.path.exists(source_code_dir): os.makedirs(source_code_dir) product_info = json_dic[project] info_list = product_info['info'] source_code = product_info['source_code'] if source_code == 'c': source = 'c' else: source = 'cpp' source_dir = os.path.join(repo_dir, source_code, project) # 工程源目录 target_dir = os.path.join(source_code_dir, project) # 工程现目录 #判断是否已经建立 # if os.path.exists(os.path.join(source_code_dir,'manifest.xml')): # continue # else: if os.path.exists(target_dir): # shutil.rmtree(target_dir) pass else: print('start copy project files ...') shutil.copytree(source_dir, target_dir) print('copy successful !!') container = ET.Element('container') repo = Repo(target_dir) r = Git(target_dir) for info in info_list: #遍历每个commit commit_id = info['commit_id'] cve_info = info['cve_info'] cwe_list = re.findall('cwe-[0-9]+', str(cve_info), re.I) try: for detail in RepositoryMining( target_dir, single=commit_id).traverse_commits(): submissionDate = detail.committer_date.strftime('%Y-%m-%d') author = detail.author # 创建testcase 节点 testcase = ET.Element('testcase') testcaseId = commit_id testcase.set('id', testcaseId) testcase.set('type', s_type) testcase.set('status', status) testcase.set('submissionDate', submissionDate) testcase.set('language', source) testcase.set('author', author.name) # 创建testcase 节点 # 创建description节点 description = ET.Element('description') description.text = detail.msg testcase.append(description) # 创建description节点 # 创建CVE节点 for cve_id in cve_info: cve = ET.Element('CVE') cve.set('id', cve_id) testcase.append(cve) #创建CVE节点 #创建CWE节点 for cwe in cwe_list: cwe_ = ET.Element('CWE') cwe_.set('id', cwe) testcase.append(cwe_) # 创建CWE节点 # 创建file节点 flaw_file_new_path = [ file.new_path.replace('\\', '/') for file in detail.modifications if file.new_path ] flaw_file_old_path = [ file.old_path.replace('\\', '/') for file in detail.modifications if file.old_path ] file_set = set(flaw_file_new_path + flaw_file_old_path) #遍历与commit相关的文件 for file in detail.modifications: diff_parsed = file.diff_parsed del_lines = [i[0] for i in diff_parsed['deleted']] add_lines = [i[0] for i in diff_parsed['added']] if file.new_path: new_path = file.new_path.replace('\\', '/') # 创建new_file节点 new_file_node = ET.Element('file') new_file_node.set('path', new_path) new_file_node.set('language', source) if len(del_lines) > 0: for line in del_lines: flaw = ET.Element('flaw') flaw.set('type', str(1)) flaw.set('line', str(line)) new_file_node.append(flaw) testcase.append(new_file_node) # 创建new_file节点 if file.old_path: old_path = file.old_path.replace('\\', '/') # 创建old_file节点 old_file_node = ET.Element('file') old_file_node.set('path', old_path) old_file_node.set('language', source) if len(add_lines) > 0: for line in add_lines: flaw = ET.Element('flaw') flaw.set('type', str(0)) flaw.set('line', str(line)) old_file_node.append(flaw) testcase.append(old_file_node) # 创建old_file节点 # testcase 添加 file #遍历与commit相关的文件 #遍历与commit不相关的文件 past_branch_name = project repo.create_head(past_branch_name, commit_id) try: r.execute('git checkout ' + past_branch_name + ' -f', shell=True) except: r.execute('git branch -D ' + past_branch_name, shell=True) r.execute('git checkout ' + past_branch_name + ' -f', shell=True) not_flaw_files = [ os.path.relpath(file, target_dir).replace('\\', '/') for file in GitRepository(target_dir).files() ] print(len(not_flaw_files), project) try: r.execute('git checkout master -f', shell=True) except: r.execute('git checkout unstable -f', shell=True) r.execute('git branch -D ' + past_branch_name, shell=True) # get the list of files present in the repo at the current commit print('write file ....') for file in not_flaw_files: if file not in file_set: if ( not file.endswith(".Cpp") and not file.endswith(".c++") and not file.endswith(".C") and not file.endswith(".cxx") and not file.endswith(".c") and not file.endswith(".cpp") and not file.endswith(".cc") and not file.endswith(".h") ): # filter files that do not contain source code continue file_node = ET.Element('file') file_node.set('path', file) file_node.set('language', source) testcase.append(file_node) #遍历与commit不相关的文件 # container 添加 testcase container.append(testcase) except Exception as e: # with open('log.txt','a+') as f: # f.write(target_dir +'\n'+commit_id+'\n'+ # str(e)+'\n'+str(time.strftime('%Y-%m-%d %H:%M:%S'))+'\n') print(e) # 写入为manifest.xml print('writing ' + project + ' manifest.xml') prettyXml(container, '\t', '\n') # 执行美化方法 # ET.dump(root) # 显示出美化后的XML内容 tree = ET.ElementTree(container) man_dir = os.path.join(source_code_dir, 'manifest.xml') tree.write(man_dir, 'utf-8', True) print('write successful!') print('parse successful!')
# In[4]: local = get_com_dict(r) # local # In[5]: github = get_com_dict(r, origin=True) # github # In[6]: github_dt = github[0]['Date'] local_dt = local[0]['Date'] # In[7]: if github_dt > local_dt: print('从github拉取') r.execute('git pull', shell=True) elif github_dt < local_dt: print('同步到github') r.execute('git push', shell=True) else: print('代码已经同步') print('完成') # In[ ]:
from git import Repo, Git import io r = Repo(path=".") l = [ item for item in r.index.diff(None) if item.a_path.endswith(".dat") and item.change_type == "M" ] len(l) for i in l: print(i.a_path) g = Git(Repo.git_dir) with io.StringIO( g.execute(["git", "--no-pager", "show", f"HEAD~1:{i.a_path}"])) as old: with open(i.a_path, mode="r+") as new: with io.StringIO(new.read()) as copy: new.seek(0) line = old.readline() while line and line.startswith("#"): new.write(line) line = old.readline() line = copy.readline() end = False while line: if not end and not line.startswith("#"): end = True if end: new.write(line) line = copy.readline() new.truncate()
class _GitWrapperCommon(object): ''' Wrap git module to provide a more stable interface across versions ''' def __init__(self, repo_path): self.git = Git() self.git.update_environment(GIT_CONFIG_NOSYSTEM='true', HOME=os.getcwd(), XDG_CONFIG_HOME=os.getcwd()) self.repo = Repo(os.path.abspath('.')) def is_file_managed_by_git(self, path): ''' :param path: Path to check :returns: True if path is managed by git ''' status, _stdout, _stderr = self.git.execute( ['git', 'ls-files', path, '--error-unmatch'], with_extended_output=True, with_exceptions=False) return status == 0 def is_file_modified(self, path): ''' Does a file have local changes not yet committed :returns: True if file has local changes ''' status, _stdout, _stderr = self.git.execute( ['git', 'diff', '--quiet', 'HEAD', path], with_extended_output=True, with_exceptions=False) return status != 0 def get_commits_following(self, path): ''' Get all commits including path following the file through renames :param path: Path which we will find commits for :returns: Sequence of commit objects. Newest to oldest ''' return [commit for commit, _ in self.get_commits_and_names_iter(path)] def get_commits_and_names_iter(self, path): ''' Get all commits including a given path following renames ''' log_result = self.git.log('--pretty=%H', '--follow', '--name-only', '--', path).splitlines() for commit_sha, _, filename in grouper(log_result, 3): yield self.repo.commit(commit_sha), filename def get_commits(self, path, follow=False): ''' Get all commits including path :param path: Path which we will find commits for :param bool follow: If True we will follow path through renames :returns: Sequence of commit objects. Newest to oldest ''' if follow: return self.get_commits_following(path) else: return self._get_commits(path)
#!/usr/bin/env python3 from git import Repo,Git import io r=Repo(path=".") l=[item for item in r.index.diff("HEAD") if item.a_path.endswith(".dat") and item.change_type=="M"] for i in l: print(i.a_path) g=Git(Repo.git_dir) with io.StringIO(g.execute(["git" ,"--no-pager", "show", f"HEAD~1:{i.a_path}"])) as old: with open(i.a_path,mode="r+") as new: with io.StringIO(new.read()) as copy: new.seek(0) line=old.readline() while line and line.startswith("#"): new.write(line) line=old.readline() line=copy.readline() end=False while line: if not end and not line.startswith("#"): end=True if end: new.write(line) line=copy.readline() new.truncate()
pass df = pd.read_csv('tomcat.csv', sep='|') bug_ids = df['bug_id'].dropna().astype(int).tolist() git = Git('C:\\Users\\aqd14\\Documents\\GitHub\\tomcat70') # p = re.compile('commit \w+') try: with open('info.json', mode='w') as f: d = dict() for id in bug_ids: output = git.execute( command= 'git log --all --grep=https://bz.apache.org/bugzilla/show_bug.cgi?id={}' .format(id)) print('id = {}'.format(id)) if not output: print('This might not be a fix for bug id = {}'.format(id)) continue # print(output) commit_ids = extract_commit_ids(output) # print(commit_ids) revisions = extract_revisions(output) # print(revisions) class_info = get_class_changed(commit_ids, revisions) d[id] = class_info json.dump(d, fp=f, indent=4)
class GitService: def __init__(self, work_dir): self._work_dir = work_dir self._git = Git(work_dir) self._remote_url = None self._name = None self._domain = None self._user = None def init_from_remote(self): self._remote_url = self._git.execute("git remote get-url origin") matcher = re.match(r"git@(?P<domain>.+):(?P<user>.+)/(?P<name>.+).git", self._remote_url) self._name = matcher.group("name") self._domain = matcher.group("domain") self._user = matcher.group("user") def get_name(self): return self._name def get_ssh_url(self): return self._remote_url def get_http_url(self): return f"https://{self._domain}/{self._user}/{self._name}" def is_github(self): return self._domain == "github.com" def get_repo_id(self): return f"{self._user}/{self._name}" def init(self, ssh_url, auto_commit_all=False): repo = Repo.init(self._work_dir) repo.create_remote('origin', ssh_url) if auto_commit_all: self._git.execute('git add -A') else: readme = Path(self._work_dir + '/README.md') if not readme.exists(): readme.touch() repo.index.add('README.md') repo.index.commit('init') self._git.execute("git push --set-upstream origin master") print("Created local repo") def from_template(self, target_ssh_url, template_ssh_url='[email protected]:AlexanderWyss/Web-Starter.git'): self._git.execute(f"git clone {template_ssh_url} .") shutil.rmtree(self._work_dir + '/.git', ignore_errors=True) self.init(target_ssh_url, True) print("Created Repo from templated") def commit(self, message): self._git.execute('git add -A') self._git.execute(f'git commit -m "{message}"') self._git.execute('git push')
if('name' in node and (master or node['name']!='master')): branches.append(node['name']) return sorted(branches, reverse=True) repos_data = getJsonData(JSON_URL, 'repos.json') packet_list = getPackets(repos_data) for packet_name in packet_list: packet = getPacket(repos_data, packet_name) if(packet==None): continue branches = getBranches(packet) if(branches==None or len(branches)<1): continue git_url = getGitUrl(packet) if(git_url == None): continue directory = (OUTPUT_DIRECTORY + packet_name) checkDirectory(directory) command = "git clone \'%s\' \'%s\' -b \'%s\'" % (git_url, directory, branches[0]) message = "\nGetting branch %s: \n\tfrom: %s\n\tto : %s\n\texec: %s" % (branches[0], git_url, directory, command) print(message) #call(['git', 'clone', git_url, directory, '-b', branches[0]]) g = Git( directory ) g.execute(['git', 'clone', git_url, directory, '-b', branches[0]])
def main(): """ The algorithm is roughly: 1. Is the current HEAD associated with a tag that looks like a release version? 2. If "yes" then use that as the version 3. If "no" then is the current branch master? 4. If "yes" the current branch is master, then inspect the branches that fit the convention for a release branch and choose the latest; increment the minor version, append .0 to form the new version (e.g., releases/v3.3 becomes 3.4.0), and append a pre-release marker 5. If "no" the current branch is not master, then determine the most recent tag in history; strip any pre-release marker, increment the patch version, and append a new pre-release marker """ repo = Repo('.') assert not repo.bare head_tag_ver = check_head_tag(repo) if head_tag_ver: return head_tag_ver version_loose = LooseVersion('0.0.0') prerelease_marker = datetime.date.today().strftime('%Y%m%d') \ + '+git' + repo.head.commit.hexsha[:10] if DEBUG: print 'Calculating release version for branch: ' + repo.active_branch.name if repo.active_branch.name == 'master': version_new = {} # Use refs (not branches) to get local branches plus remote branches for ref in repo.refs: release_branch_match = RELEASE_BRANCH_RE.match(ref.name) if release_branch_match: # Construct a candidate version from this branch name version_new['major'] = int(release_branch_match.group('vermaj')) version_new['minor'] = int(release_branch_match.group('vermin')) + 1 version_new['patch'] = 0 version_new['prerelease'] = prerelease_marker new_version_loose = LooseVersion(str(version_new['major']) + '.' + str(version_new['minor']) + '.' + str(version_new['patch']) + '-' + version_new['prerelease']) if new_version_loose > version_loose: version_loose = new_version_loose if DEBUG: print 'Found new best version "' + str(version_loose) \ + '" on branch "' + ref.name + '"' else: gexc = Git('.') tags = gexc.execute(['git', 'tag', '--merged', 'HEAD', '--list', 'r*', '--sort', 'version:refname']) if len(tags) > 0: release_tag_match = RELEASE_TAG_RE.match(tags.splitlines()[-1]) if release_tag_match: version_new = {} version_new['major'] = int(release_tag_match.group('vermaj')) version_new['minor'] = int(release_tag_match.group('vermin')) version_new['patch'] = int(release_tag_match.group('verpatch')) + 1 version_new['prerelease'] = prerelease_marker new_version_loose = LooseVersion(str(version_new['major']) + '.' + str(version_new['minor']) + '.' + str(version_new['patch']) + '-' + version_new['prerelease']) if new_version_loose > version_loose: version_loose = new_version_loose if DEBUG: print 'Found new best version "' + str(version_loose) \ + '" from tag "' + release_tag_match.group('ver') + '"' return str(version_loose)
class _GitWrapperCommon(object): ''' Wrap git module to provide a more stable interface across versions ''' def __init__(self, repo_path): self.git = Git() self.git.update_environment( GIT_CONFIG_NOSYSTEM='true', HOME=os.getcwd(), XDG_CONFIG_HOME=os.getcwd() ) self.repo = Repo(os.path.abspath('.')) def is_file_managed_by_git(self, path): ''' :param path: Path to check :returns: True if path is managed by git ''' status, _stdout, _stderr = self.git.execute( ['git', 'ls-files', path, '--error-unmatch'], with_extended_output=True, with_exceptions=False) return status == 0 def is_file_modified(self, path): ''' Does a file have local changes not yet committed :returns: True if file has local changes ''' status, _stdout, _stderr = self.git.execute( ['git', 'diff', '--quiet', 'HEAD', path], with_extended_output=True, with_exceptions=False) return status != 0 def get_commits_following(self, path): ''' Get all commits including path following the file through renames :param path: Path which we will find commits for :returns: Sequence of commit objects. Newest to oldest ''' return [ commit for commit, _ in self.get_commits_and_names_iter( path)] def get_commits_and_names_iter(self, path): ''' Get all commits including a given path following renames ''' log_result = self.git.log( '--pretty=%H', '--follow', '--name-only', '--', path).splitlines() for commit_sha, _, filename in grouper(log_result, 3): yield self.repo.commit(commit_sha), filename def get_commits(self, path, follow=False): ''' Get all commits including path :param path: Path which we will find commits for :param bool follow: If True we will follow path through renames :returns: Sequence of commit objects. Newest to oldest ''' if follow: return self.get_commits_following(path) else: return self._get_commits(path)