def show_context(filename, line_number, show_line=3, is_back=False):
if not show_line:
return ""
filename = check_filepath(PROJECT_DIRECTORY, filename)
line_number = line_number if line_number else 0
line_start = int(line_number) - show_line if (int(line_number) -
show_line) > 0 else 0
line_start = line_start if line_start else 1
line_end = int(line_start) + show_line + show_line
lines = get_line(filename, "{},{}".format(line_start, line_end))
contents = ""
i = 0
for line in lines:
if not is_back:
if line_start + i == int(line_number):
logger_console.warning(
"%4d: %s" % (line_start + i, line.replace("\n", "")))
else:
logger_console.info("%4d: %s" %
(line_start + i, line.replace("\n", "")))
contents += "%4d: %s" % (line_start + i, line)
i += 1
return contents
def show_context(filename, line_number, show_line=3):
filename = check_filepath(PROJECT_DIRECTORY, filename)
line_start = int(line_number) - show_line
line_end = int(line_number) + show_line
lines = get_line(filename, "{},{}".format(line_start, line_end))
i = 0
for line in lines:
i += 1
if i == (show_line + 1):
logger_console.warning(
"%4d: %s" % (line_start + i - 1, line.replace("\n", "")))
else:
logger_console.info("%4d: %s" %
(line_start + i - 1, line.replace("\n", "")))
def check_vendor(self):
for file in self.exist_file_list:
try:
filepath = check_filepath(self.target_path, file)
filename = file.split('/')[-1].split('\\')[-1]
language = self.get_language(filename)
f = codecs.open(filepath,
'rb+',
encoding='utf-8',
errors='ignore')
filecontent = f.read()
f.seek(0, os.SEEK_SET)
if filename == "requirements.txt":
for line in f:
if not len(line):
continue
vendor = line.split("==")
vendor_name = vendor[0].strip()
vendor_version = vendor[-1].strip()
if len(vendor) < 2:
vendor_version = None
update_and_new_project_vendor(self.project_id,
name=vendor_name,
version=vendor_version,
language=language)
elif filename == 'composer.json':
vendors = json.loads(filecontent, encoding='utf-8')
if not len(vendors):
continue
vendors_list = vendors['require']
for vendor in vendors_list:
vendor_name = vendor.strip()
vendor_version = vendors_list[vendor].strip()
update_and_new_project_vendor(self.project_id,
name=vendor_name,
version=vendor_version,
language=language)
elif filename == 'go.mod':
go_version = ""
is_require_line = False
for line in f:
if line.startswith('go'):
go_version = line.strip().split(' ')[-1]
if line.startswith('require ('):
is_require_line = True
continue
if line.startswith(')'):
is_require_line = False
continue
if is_require_line:
vendor = self.check_commit(line).strip().split(' ')
vendor_name = vendor[0].strip()
vendor_version = vendor[-1].strip()
update_and_new_project_vendor(
self.project_id,
name=vendor_name,
version=vendor_version,
language=language,
ext=go_version)
elif filename == 'pom.xml':
reg = r'xmlns="([\w\.\\/:]+)"'
pom_ns = None
if re.search(reg, filecontent, re.I):
p = re.compile(reg)
matchs = p.finditer(filecontent)
for match in matchs:
pom_ns = match.group(1)
if pom_ns:
xpath_reg = ".//{%s}dependency" % pom_ns
else:
xpath_reg = ".//dependency"
tree = self.parse_xml(filepath)
root = tree.getroot()
childs = root.findall(xpath_reg)
for child in childs:
group_id = child.getchildren()[0].text
artifact_id = child.getchildren()[1].text
if len(child.getchildren()) > 2:
version = child.getchildren()[2].text
else:
version = 'latest'
var_reg = "\${([\w\.\_-]+)}"
if re.search(var_reg, version, re.I):
p2 = re.compile(var_reg)
matchs = p2.finditer(version)
for match in matchs:
varname = match.group(1)
if pom_ns:
var_xpath_reg = ".//{%s}%s" % (pom_ns,
varname)
else:
var_xpath_reg = ".//%s" % varname
varchilds = root.findall(var_xpath_reg)
for child in varchilds:
version = child.text
vendor_name = "{}:{}".format(group_id, artifact_id)
vendor_version = version
ext = "maven"
update_and_new_project_vendor(self.project_id,
name=vendor_name,
version=vendor_version,
language=language,
ext=ext)
elif filename == 'build.gradle':
is_plugin_block = False
ext = "gradle"
for line in f:
if line.startswith('plugins {'):
is_plugin_block = True
continue
if is_plugin_block and line.startswith('}'):
is_plugin_block = False
continue
if is_plugin_block:
plugin_block_list = line.strip().split(' ')
last_block = ""
vendor_name = ""
vendor_version = ""
for plugin_block in plugin_block_list:
if last_block == 'id':
vendor_name = plugin_block.strip(
"'").strip('"')
if last_block == 'version':
vendor_version = plugin_block.strip(
"'").strip('"')
last_block = plugin_block
if vendor_name and vendor_version:
update_and_new_project_vendor(
self.project_id,
name=vendor_name,
version=vendor_version,
language=language,
ext=ext)
continue
elif filename == "package.json":
vendors = json.loads(filecontent, encoding='utf-8')
if not len(vendors):
continue
node_version = "{} {}".format(vendors['name'],
vendors['version'])
dependencies = vendors["dependencies"]
devDependencies = vendors["devDependencies"]
for dependency in dependencies:
vendor_version = dependencies[dependency].strip()
ext = "{}.{}".format(node_version, "dependencies")
update_and_new_project_vendor(self.project_id,
name=dependency,
version=vendor_version,
language=language,
ext=ext)
for dependency in devDependencies:
vendor_version = devDependencies[dependency].strip()
ext = "{}.{}".format(node_version, "devDependencies")
update_and_new_project_vendor(self.project_id,
name=dependency,
version=vendor_version,
language=language,
ext=ext)
else:
logger.warn(
"[Vendor] Vendor file {} not support".format(filename))
except:
logger.error(
"[Vendor] Error check for Vendor file {}.\nError: {}".
format(file, traceback.format_exc()))
continue
def check_vendor(self):
for file in self.exist_file_list:
try:
filepath = check_filepath(self.target_path, file)
filename = file.split('/')[-1].split('\\')[-1]
language = self.get_language(filename)
f = codecs.open(filepath, 'rb+', encoding='utf-8', errors='ignore')
filecontent = f.read()
f.seek(0, os.SEEK_SET)
savefilepath = filepath.replace(self.target_path, "").replace('\\', '/')
logger.info("[Vendor] Parse File {}.".format(savefilepath))
if filename == "requirements.txt":
for line in f:
if not len(line):
continue
vendor = line.split("==")
vendor_name = vendor[0].strip()
vendor_version = vendor[-1].strip()
if len(vendor) < 2:
vendor_version = None
ext = "php"
update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
language=language, source=savefilepath, ext=ext)
get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language)
elif filename == 'composer.json':
vendors = json.loads(filecontent)
vendors_list = []
ext = "php"
if not len(vendors):
continue
if 'require' in vendors:
vendors_list = vendors['require']
for vendor in vendors_list:
vendor_name = vendor.strip()
vendor_version = vendors_list[vendor].strip()
update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
language=language, source=savefilepath, ext=ext)
get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language)
elif filename == 'go.mod':
go_version = ""
is_require_line = False
for line in f:
if line.startswith('go'):
go_version = line.strip().split(' ')[-1]
if line.startswith('require ('):
is_require_line = True
continue
if line.startswith(')'):
is_require_line = False
continue
if is_require_line:
vendor = self.check_commit(line).strip().split(' ')
vendor_name = vendor[0].strip()
vendor_version = vendor[-1].strip()
ext = go_version
update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
language=language, source=savefilepath, ext=ext)
get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language)
elif filename == 'pom.xml':
reg = r'xmlns="([\w\.\\/:]+)"'
pom_ns = None
ext = ""
if re.search(reg, filecontent, re.I):
p = re.compile(reg)
matchs = p.finditer(filecontent)
for match in matchs:
pom_ns = match.group(1)
tree = self.parse_xml(filepath)
root = tree.getroot()
# 匹配default
if pom_ns:
default_xpath_reg = ".//{%s}parent" % pom_ns
else:
default_xpath_reg = ".//parent"
parents = root.findall(default_xpath_reg)
default_version = "unknown"
project_version = "unknown"
for parent in parents:
project_groupid = parent.getchildren()[0].text
project_artifactId = parent.getchildren()[1].text
project_version = parent.getchildren()[2].text
# project version 格式检查
var_reg = "\${([\w\.\_-]+)}"
if re.search(var_reg, project_version, re.I):
p2 = re.compile(var_reg)
matchs = p2.finditer(project_version)
for match in matchs:
varname = match.group(1)
if varname in self.java_temp_vendor_list:
project_version = self.java_temp_vendor_list[varname]
continue
# project 依赖版本也可以加入全局表
vendor_name = "{}.{}".format(project_groupid, project_artifactId)
self.java_temp_vendor_list[vendor_name] = project_version
update_and_new_project_vendor(self.project_id, name=vendor_name, version=project_version,
language=language, source=savefilepath, ext=ext)
# 匹配通用配置
if pom_ns:
java_base_xpath_reg = ".//{%s}properties" % pom_ns
else:
java_base_xpath_reg = ".//properties"
base_tags = root.findall(java_base_xpath_reg)
if base_tags:
btags = base_tags[0].getchildren()
for btag in btags:
self.java_temp_vendor_list[btag.tag.replace("{%s}" % pom_ns, "")] = btag.text
# 全局表
vendor_name = btag.tag.replace("{%s}" % pom_ns, "")
self.java_temp_vendor_list[vendor_name] = btag.text
update_and_new_project_vendor(self.project_id, name=vendor_name, version=btag.text,
language=language, source=savefilepath, ext=ext)
# 匹配dependency
if pom_ns:
xpath_reg = ".//{%s}dependency" % pom_ns
else:
xpath_reg = ".//dependency"
childs = root.findall(xpath_reg)
for child in childs:
group_id = child.getchildren()[0].text
artifact_id = child.getchildren()[1].text
if len(child.getchildren()) > 2 and "version" in child.getchildren()[2].tag:
version = child.getchildren()[2].text
else:
version = default_version
var_reg = "\${([\w\.\_-]+)}"
if re.search(var_reg, version, re.I):
p2 = re.compile(var_reg)
matchs = p2.finditer(version)
for match in matchs:
varname = match.group(1)
# 处理内置变量
if varname == "project.version":
version = project_version
continue
if varname in self.java_temp_vendor_list:
version = self.java_temp_vendor_list[varname]
continue
# if pom_ns:
# var_xpath_reg = ".//{%s}%s" % (pom_ns, varname)
# else:
# var_xpath_reg = ".//%s" % varname
#
# varchilds = root.findall(var_xpath_reg)
# for child in varchilds:
# version = child.text
# ext = varname
#
# # 如果没有匹配到，那么需要去数据库查询
# if not varchilds:
# pv = ProjectVendors.objects.filter(project_id=self.project_id, ext=varname).first()
# if pv:
# version = pv.version
vendor_name = "{}:{}".format(group_id, artifact_id)
vendor_version = version
ext = "mevan"
logger.debug("[Vendor][pom.xml] Found Vendor {} vension {} in file {}".format(vendor_name, vendor_version, savefilepath))
update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
language=language, source=savefilepath, ext=ext)
get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext)
elif filename == 'build.gradle':
is_plugin_block = False
ext = "gradle"
for line in f:
if line.startswith('plugins {'):
is_plugin_block = True
continue
if is_plugin_block and line.startswith('}'):
is_plugin_block = False
continue
if is_plugin_block:
plugin_block_list = line.strip().split(' ')
last_block = ""
vendor_name = ""
vendor_version = ""
for plugin_block in plugin_block_list:
if last_block == 'id':
vendor_name = plugin_block.strip("'").strip('"')
if last_block == 'version':
vendor_version = plugin_block.strip("'").strip('"')
last_block = plugin_block
if vendor_name and vendor_version:
update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version,
language=language, source=savefilepath, ext=ext)
get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext)
continue
elif filename == "package.json":
vendors = json.loads(filecontent)
if not len(vendors):
continue
node_version = "{} {}".format(vendors['name'], vendors['version'])
dependencies = vendors["dependencies"]
devDependencies = vendors["devDependencies"]
for dependency in dependencies:
vendor_version = dependencies[dependency].strip()
ext = "{}.{}".format(node_version, "dependencies")
update_and_new_project_vendor(self.project_id, name=dependency, version=vendor_version,
language=language, source=savefilepath)
get_and_save_vendor_vuls(self.task_id, dependency, vendor_version, language, ext)
for dependency in devDependencies:
vendor_version = devDependencies[dependency].strip()
ext = "{}.{}".format(node_version, "devDependencies")
update_and_new_project_vendor(self.project_id, name=dependency, version=vendor_version,
language=language, source=savefilepath)
get_and_save_vendor_vuls(self.task_id, dependency, vendor_version, language, ext)
else:
logger.warn("[Vendor] Vendor file {} not support".format(filename))
except:
logger.error("[Vendor] Error check for Vendor file {}.\nError: {}".format(file, traceback.format_exc()))
continue