def main():
parser=OptionParser(usage=__doc__)
parser.add_option("-l","--iscc-binary", dest="iscc_binary", default=os.path.join(os.path.dirname(__file__),'innosetup','ISCC.exe'), help="Path to ISCC compiler (default: %default)")
parser.add_option("-v","--wapt-version", dest="waptversion", default=waptutils.__version__, help="Wapt edition to build (community, enterprise...). (default: %default)")
parser.add_option("-e","--wapt-edition", dest="waptedition", default='community', help="Wapt edition to build (community, enterprise...). (default: %default)")
parser.add_option("-k","--sign-key", dest="sign_key_path", help="Sign with this key. (default: %default)")
parser.add_option("-w","--sign-key-pwd-path", dest="sign_key_pwd_path", help="Path to password file. (default: %default)")
parser.add_option("-x","--sign-exe-filenames", dest="exe_filenames", help="Additional executables to sign. (default: %default)")
(options,args) = parser.parse_args()
if len(args) != 1:
parser.usage
sys.exit(1)
for iss_path in args:
iss_path = os.path.abspath(iss_path)
(iss_rootname,issext) = os.path.splitext(iss_path)
# add a revision.txt file with git short
r = Repo(search_parent_directories=True)
rev_file = open(os.path.join(os.path.dirname(iss_path), '..', 'revision.txt'), 'w')
rev_file.write(r.head.object.hexsha[:8])
rev_file.close()
r.close()
iss_file = iss_rootname + ".iss"
if options.sign_key_path and options.exe_filenames:
exes = options.exe_filenames.split(',')
for exe_fn in exes:
sign_exe(exe_fn,options.sign_key_path,open(options.sign_key_pwd_path,'rb').read())
cmd = '"%(issc_binary)s" /Dwapt%(waptedition)s %(issfile)s' % {
'issc_binary':options.iscc_binary,
'issfile':iss_file,
'waptedition':options.waptedition.lower()
}
res = setuphelpers.run(cmd)
exe_fn = res.splitlines()[-1]
if options.sign_key_path:
sign_exe(exe_fn,options.sign_key_path,open(options.sign_key_pwd_path,'rb').read())
def main():
parser=OptionParser(usage=__doc__)
parser.add_option("-l","--iscc-binary", dest="iscc_binary", default=os.path.join(os.path.dirname(__file__),'innosetup','ISCC.exe'), help="Path to ISCC compiler (default: %default)")
parser.add_option("-v","--wapt-version", dest="waptversion", default=waptutils.__version__, help="Wapt edition to build (community, enterprise...). (default: %default)")
parser.add_option("-e","--wapt-edition", dest="waptedition", default='community', help="Wapt edition to build (community, enterprise...). (default: %default)")
parser.add_option("-k","--sign-key", dest="sign_key_path", help="Sign with this key. (default: %default)")
parser.add_option("-w","--sign-key-pwd-path", dest="sign_key_pwd_path", help="Path to password file. (default: %default)")
parser.add_option("-x","--sign-exe-filenames", dest="exe_filenames", help="Additional executables to sign. (default: %default)")
(options,args) = parser.parse_args()
if len(args) != 1:
parser.usage
sys.exit(1)
for iss_path in args:
iss_path = os.path.abspath(iss_path)
(iss_rootname,issext) = os.path.splitext(iss_path)
# add a revision.txt file with git short
r = Repo(search_parent_directories=True)
rev_file = open(os.path.join(os.path.dirname(iss_path), '..', 'revision.txt'), 'w')
rev_file.write(r.head.object.hexsha[:8])
rev_file.close()
r.close()
iss_file = iss_rootname + ".iss"
if options.sign_key_path and options.exe_filenames:
exes = options.exe_filenames.split(',')
for exe_fn in exes:
sign_exe(exe_fn,options.sign_key_path,open(options.sign_key_pwd_path,'rb').read())
cmd = '"%(issc_binary)s" /Dwapt%(waptedition)s %(issfile)s' % {
'issc_binary':options.iscc_binary,
'issfile':iss_file,
'waptedition':options.waptedition.lower()
}
res = setuphelpers.run(cmd)
exe_fn = res.splitlines()[-1]
if options.sign_key_path:
sign_exe(exe_fn,options.sign_key_path,open(options.sign_key_pwd_path,'rb').read())
def find_changed_code_commit(patch_link, project_dir, apis_file, total_api):
# look up the commitid in the commit page
try:
patch_req = requests.get(patch_link, timeout=40)
except:
raise RuntimeError('request timeout')
patch_soup = BeautifulSoup(patch_req.text, "lxml")
commitid = patch_soup.find(
'div',
class_='flex-auto no-wrap text-lg-right text-left overflow-x-auto'
).find('span', class_='sha user-select-contain').text
message_all = patch_soup.find_all(
'div',
class_=
'file js-file js-details-container js-targetable-element Details Details--on open show-inline-notes'
)
is_error = 0
try:
# change the project version
repo = Repo(project_dir)
repo.git.checkout(commitid)
repo.close()
print('-----successfully change to commit: ' + commitid + '-----')
except:
info = {}
info['error'] = 'change to commit ' + commitid + ' failed'
jsonData = json.dumps(info, ensure_ascii=False)
apis_file.write(jsonData)
apis_file.write('\n')
is_error = 1
if not is_error:
# create and open an udb
udbfile = project_dir + '/' + project_dir.split('/')[-1] + '.udb'
os.system(
'/home/user/scitools/bin/linux64/und create -languages Java ' +
udbfile)
os.system('/home/user/scitools/bin/linux64/und add ' + project_dir +
' ' + udbfile)
os.system('/home/user/scitools/bin/linux64/und analyze ' + udbfile)
db = understand.open(udbfile)
# look up changed files and lines in commit page
for message in message_all:
changed_lines = []
file_name = message.find(
'div',
class_='file-info flex-auto min-width-0 mb-md-0 mb-2').find(
'a').text
if file_name.find('.java') == -1:
continue
message1_all = message.find(
'div', class_='data highlight js-blob-wrapper').find_all(
'td',
class_='blob-num blob-num-addition js-linkable-line-number'
)
if message1_all == None:
continue
for message1 in message1_all:
message2 = message1['data-line-number']
changed_lines.append(int(message2))
if changed_lines == []:
continue
print('-----find changed_lines in file : ' + file_name + '-----')
# use understand tool to analyze the vulnerability in this file
find_vulnerable_api(db, file_name, changed_lines, apis_file,
total_api, commitid)
print('-----vulnerable_api analyze done-----')
db.close()
os.remove(udbfile)
def find_changed_code_pull(pull_link, project_dir, apis_file, total_api):
commits_link = pull_link + '/commits'
# get commit page in commits page
try:
commits_req = requests.get(commits_link, timeout=40)
except:
raise RuntimeError('request timeout')
commits_soup = BeautifulSoup(commits_req.text, "lxml")
commits_msg = commits_soup.find(
'div',
class_=
'commits-listing commits-listing-padded js-navigation-container js-active-navigation-container'
).find_all('li')
for commit_msg in commits_msg:
commit_href = commit_msg.find('div', class_='table-list-cell').find(
'a', class_='message js-navigation-open')['href']
patch_link = 'https://github.com' + commit_href
# look up the commitid in the commit page
try:
patch_req = requests.get(patch_link, timeout=40)
except:
raise RuntimeError('request timeout')
patch_soup = BeautifulSoup(patch_req.text, "lxml")
commitid = patch_soup.find(
'div',
class_='commit-meta clearfix p-2 no-wrap d-flex flex-items-center'
).find('span', class_='sha user-select-contain').text
message_all = patch_soup.find_all(
'div',
class_=
'file js-file js-details-container js-targetable-element Details Details--on open show-inline-notes'
)
try:
# change the project version
repo = Repo(project_dir)
repo.git.checkout(commitid)
repo.close()
print('-----successfully change to commit: ' + commitid + '-----')
except:
info = {}
info['error'] = 'change to commit ' + commitid + ' failed'
jsonData = json.dumps(info, ensure_ascii=False)
apis_file.write(jsonData)
apis_file.write('\n')
continue
# create and open an udb
udbfile = project_dir + '/' + project_dir.split('/')[-1] + '.udb'
print(udbfile)
os.system(
'/home/user/scitools/bin/linux64/und create -languages Java ' +
udbfile)
os.system('/home/user/scitools/bin/linux64/und add ' + project_dir +
' ' + udbfile)
os.system('/home/user/scitools/bin/linux64/und analyze ' + udbfile)
db = understand.open(udbfile)
# look up changed files and lines in commit page
for message in message_all:
changed_lines = []
file_name = message.find(
'div', class_='file-info flex-auto').find('a').text
if file_name.find('.java') == -1:
continue
message1_all = message.find(
'div', class_='data highlight js-blob-wrapper').find_all(
'td',
class_='blob-num blob-num-addition js-linkable-line-number'
)
message2_all = message.find(
'div', class_='data highlight js-blob-wrapper').find_all(
'span',
attrs={
'class': 'blob-code-inner blob-code-marker',
'data-code-marker': '+'
})
if message1_all == None:
continue
for k in range(len(message1_all)):
# if it's a blank line
if message2_all[k].find('span') == None:
continue
# if it's a comment
if message2_all[k].find('span', class_='pl-c') != None:
continue
message3 = message1_all[k]['data-line-number']
changed_lines.append(int(message3))
print('-----find changed_lines in file : ' + file_name + '-----')
# use understand tool to analyze the vulnerability in this file
find_vulnerable_api(db, file_name, changed_lines, apis_file,
total_api, commitid)
print('-----vulnerable_api analyze done-----')
db.close()
os.remove(udbfile)
class GitArchiver(BaseArchiver):
"""Gitpython implementation of the base archiver."""
name = "git"
def __init__(self, config):
"""
Instantiate a new Git Archiver.
:param config: The wily configuration
:type config: :class:`wily.config.WilyConfig`
"""
try:
self.repo = Repo(config.path)
except git.exc.InvalidGitRepositoryError as e:
raise InvalidGitRepositoryError from e
self.config = config
if self.repo.head.is_detached:
self.current_branch = self.repo.head.object.hexsha
else:
self.current_branch = self.repo.active_branch
assert not self.repo.bare, "Not a Git repository"
def revisions(self, path: str, max_revisions: int) -> List[Revision]:
"""
Get the list of revisions.
:param path: the path to target.
:type path: ``str``
:param max_revisions: the maximum number of revisions.
:type max_revisions: ``int``
:return: A list of revisions.
:rtype: ``list`` of :class:`Revision`
"""
if self.repo.is_dirty():
raise DirtyGitRepositoryError(self.repo.untracked_files)
revisions = []
for commit in self.repo.iter_commits(self.current_branch,
max_count=max_revisions,
reverse=True):
tracked_files, tracked_dirs = get_tracked_files_dirs(
self.repo, commit)
if not commit.parents or not revisions:
added_files = tracked_files
modified_files = []
deleted_files = []
else:
added_files, modified_files, deleted_files = whatchanged(
commit, self.repo.commit(commit.hexsha + "~1"))
logger.debug(
f"For revision {commit.name_rev.split(' ')[0]} found:")
logger.debug(f"Tracked files: {tracked_files}")
logger.debug(f"Tracked directories: {tracked_dirs}")
logger.debug(f"Added files: {added_files}")
logger.debug(f"Modified files: {modified_files}")
logger.debug(f"Deleted files: {deleted_files}")
rev = Revision(
key=commit.name_rev.split(" ")[0],
author_name=commit.author.name,
author_email=commit.author.email,
date=commit.committed_date,
message=commit.message,
tracked_files=tracked_files,
tracked_dirs=tracked_dirs,
added_files=added_files,
modified_files=modified_files,
deleted_files=deleted_files,
)
revisions.append(rev)
return revisions[::-1]
def checkout(self, revision: Revision, options: Dict):
"""
Checkout a specific revision.
:param revision: The revision identifier.
:type revision: :class:`Revision`
:param options: Any additional options.
:type options: ``dict``
"""
rev = revision.key
self.repo.git.checkout(rev)
def finish(self):
"""
Clean up any state if processing completed/failed.
For git, will checkout HEAD on the original branch when finishing
"""
self.repo.git.checkout(self.current_branch)
self.repo.close()
def find(self, search: str) -> Revision:
"""
Search a string and return a single revision.
:param search: The search term.
:type search: ``str``
:return: An instance of revision.
:rtype: Instance of :class:`Revision`
"""
commit = self.repo.commit(search)
tracked_files, tracked_dirs = get_tracked_files_dirs(self.repo, commit)
if not commit.parents:
added_files = tracked_files
modified_files = []
deleted_files = []
else:
added_files, modified_files, deleted_files = whatchanged(
commit, self.repo.commit(commit.hexsha + "~1"))
return Revision(
key=commit.name_rev.split(" ")[0],
author_name=commit.author.name,
author_email=commit.author.email,
date=commit.committed_date,
message=commit.message,
tracked_files=tracked_files,
tracked_dirs=tracked_dirs,
added_files=added_files,
modified_files=modified_files,
deleted_files=deleted_files,
)
