def run(self, target=None, tid=None):
if target is None:
logging.critical("Please set --target param")
sys.exit()
if tid is None:
logging.critical("Please set --tid param")
sys.exit()
# Statistic Code
p = subprocess.Popen(['cloc', target], stdout=subprocess.PIPE)
(output, err) = p.communicate()
rs = output.split("\n")
for r in rs:
r_e = r.split()
if len(r_e) > 3 and r_e[0] == 'SUM:':
t = CobraTaskInfo.query.filter_by(id=tid).first()
if t is not None:
t.code_number = r_e[4]
try:
db.session.add(t)
db.session.commit()
logging.info("Statistic code number done")
except Exception as e:
logging.error("Statistic code number failed" +
str(e.message))
def push(self):
try:
# 为杜绝前面环节问题导致输出重复,所以推送前先检查是否已经推送过
exist_vuln = CobraResults.query.filter_by(id=self.vuln_id, status=2).count()
if exist_vuln == 0:
logging.info("已经推送过")
return False
vulns = {'info': json.dumps(self.vulnerabilities)}
response = requests.post(self.api, data=vulns)
if response.text == 'done':
logging.info('推送漏洞到第三方漏洞管理平台成功')
"""
更新漏洞状态
1. 漏洞状态是初始化(0) -> 更新(1)
2. 漏洞状态是已推送(1) -> 不更新
3. 漏洞状态是已修复(2) -> 不更新
"""
if self.vuln_id is None:
logging.warning("漏洞ID不能为空")
else:
vuln = CobraResults.query.filter_by(id=self.vuln_id).first()
if vuln.status == 0:
vuln.status = 1
db.session.add(vuln)
db.session.commit()
return True
else:
logging.critical('推送第三方漏洞管理平台失败 \r\n{0}'.format(response.text))
return False
except (requests.ConnectionError, requests.HTTPError) as e:
logging.warning("推送第三方漏洞管理平台出现异常: {0}".format(e))
return False
def notification(self, capture_path):
"""
Email notification
:param capture_path:
:return: boolean
"""
msg = MIMEMultipart()
msg['Subject'] = self.subject
msg['From'] = '{0}<{1}>'.format(self.name, self.user)
msg['To'] = self.to
with open(capture_path, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
text = MIMEText('<img src="data:image/png;base64,{0}">'.format(encoded_string), 'html')
msg.attach(text)
try:
s = smtplib.SMTP(self.host, self.port)
s.ehlo()
s.starttls()
s.ehlo()
s.login(self.user, self.password)
s.sendmail(self.user, self.to, msg.as_string())
s.quit()
return True
except SMTPException:
logging.critical('Send mail failed')
return False
def pull_code(self, branch='master'):
logging.info('Gitlab project')
# Git
if 'gitlab' in self.target:
username = config.Config('git', 'username').value
password = config.Config('git', 'password').value
else:
username = None
password = None
gg = git.Git(self.target,
branch=branch,
username=username,
password=password)
# Git Clone Error
try:
clone_ret, clone_err = gg.clone()
if clone_ret is False:
return 4001, 'Clone Failed ({0})'.format(clone_err), gg
except NotExistError:
# update project status
p = CobraProjects.query.filter_by(repository=self.target).first()
if p is not None:
if p.status == CobraProjects.get_status('on'):
p.status = CobraProjects.get_status('off')
db.session.add(p)
db.session.commit()
return 4001, 'Repository Does not exist!', gg
except AuthError:
logging.critical('Git Authentication Failed')
return 4001, 'Repository Authentication Failed', gg
return 1001, 'Success', gg
def notification(self, capture_path):
"""
Email notification
:param capture_path:
:return: boolean
"""
msg = MIMEMultipart()
msg['Subject'] = self.subject
msg['From'] = '{0}<{1}>'.format(self.name, self.user)
msg['To'] = self.to
with open(capture_path, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
text = MIMEText(
'<img src="data:image/png;base64,{0}">'.format(encoded_string),
'html')
msg.attach(text)
try:
s = smtplib.SMTP(self.host, self.port)
s.ehlo()
s.starttls()
s.ehlo()
s.login(self.user, self.password)
s.sendmail(self.user, self.to, msg.as_string())
s.quit()
return True
except SMTPException:
logging.critical('Send mail failed')
return False
def push(self):
try:
# 为杜绝前面环节问题导致输出重复,所以推送前先检查是否已经推送过
exist_vuln = CobraResults.query.filter_by(id=self.vuln_id,
status=2).count()
if exist_vuln == 0:
logging.info("已经推送过")
return False
vulns = {'info': json.dumps(self.vulnerabilities)}
response = requests.post(self.api, data=vulns)
if response.text == 'done':
logging.info('推送漏洞到第三方漏洞管理平台成功')
"""
更新漏洞状态
1. 漏洞状态是初始化(0) -> 更新(1)
2. 漏洞状态是已推送(1) -> 不更新
3. 漏洞状态是已修复(2) -> 不更新
"""
if self.vuln_id is None:
logging.warning("漏洞ID不能为空")
else:
vuln = CobraResults.query.filter_by(
id=self.vuln_id).first()
if vuln.status == 0:
vuln.status = 1
db.session.add(vuln)
db.session.commit()
return True
else:
logging.critical('推送第三方漏洞管理平台失败 \r\n{0}'.format(response.text))
return False
except (requests.ConnectionError, requests.HTTPError) as e:
logging.warning("推送第三方漏洞管理平台出现异常: {0}".format(e))
return False
def pull_code(self, branch='master'):
logging.info('Gitlab project')
# Git
if 'gitlab' in self.target:
username = config.Config('git', 'username').value
password = config.Config('git', 'password').value
else:
username = None
password = None
gg = git.Git(self.target, branch=branch, username=username, password=password)
# Git Clone Error
try:
clone_ret, clone_err = gg.clone()
if clone_ret is False:
return 4001, 'Clone Failed ({0})'.format(clone_err), gg
except NotExistError:
# update project status
p = CobraProjects.query.filter_by(repository=self.target).first()
if p is not None:
if p.status == CobraProjects.get_status('on'):
p.status = CobraProjects.get_status('off')
db.session.add(p)
db.session.commit()
return 4001, 'Repository Does not exist!', gg
except AuthError:
logging.critical('Git Authentication Failed')
return 4001, 'Repository Authentication Failed', gg
return 1001, 'Success', gg
def run(self):
capture = self.capture()
if capture is False:
logging.critical('Capture failed')
return False
# send notification
if self.notification(capture):
return True
else:
logging.critical('Notification failed')
return False
def run(self):
capture = self.capture()
if capture is False:
logging.critical('Capture failed')
return False
# send notification
if self.notification(capture):
return True
else:
logging.critical('Notification failed')
return False
def allowed_file(filename):
"""
Allowd upload file
Config Path: ./config [upload]
:param filename:
:return:
"""
config_extension = config.Config('upload', 'extensions').value
if config_extension == '':
logging.critical('Please set config file upload->directory')
sys.exit(0)
allowed_extensions = config_extension.split('|')
return '.' in filename and filename.rsplit('.', 1)[1] in allowed_extensions
def log(self, level, message, test=True):
if test:
self.data.append('[{0}] {1}'.format(level.upper(), message))
if level == 'critical':
logging.critical(message)
elif level == 'warning':
logging.warning(message)
elif level == 'info':
logging.info(message)
elif level == 'debug':
logging.debug(message)
elif level == 'error':
logging.error(message)
def log(self, level, message, test=True):
if test:
self.data.append('[{0}] {1}'.format(level.upper(), message))
if level == 'critical':
logging.critical(message)
elif level == 'warning':
logging.warning(message)
elif level == 'info':
logging.info(message)
elif level == 'debug':
logging.debug(message)
elif level == 'error':
logging.error(message)
def run(self, is_all=None, target=None, tid=None, pid=None):
if bool(is_all) is True:
logging.info('[START] Scan all projects')
scan.Scan().all()
logging.info('[END] Scan all projects')
else:
if target is None:
logging.critical("Please set --target param")
sys.exit()
if tid is not None:
task_id = tid
# Start Time For Task
t = CobraTaskInfo.query.filter_by(id=tid).first()
if t is None:
logging.critical("Task id doesn't exists.")
sys.exit()
if t.status not in [0, 1]:
logging.critical("Task Already Scan.")
sys.exit()
t.status = 1
t.time_start = int(time.time())
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
logging.error("Set start time failed" + str(e.message))
else:
task_id = None
if os.path.isdir(target) is not True:
logging.critical('Target is not directory')
sys.exit()
from engine import static
static.Static(target, task_id=task_id, project_id=pid).analyse()
def block_code(self, block_position):
"""
获取搜索区块代码
:param block_position:
0:up 上
1:down 下
2:location_line 当前行
:return:
"""
if block_position == 2:
if self.line is None or self.line == 0:
logging.critical("行号异常: {0}".format(self.line))
return False
line_rule = '{0}p'.format(self.line)
code = File(self.file_path).lines(line_rule)
code = code.strip()
return code
else:
block_start = 1
block_end = 0
functions = self.functions()
if functions:
for function_name, function_value in functions.items():
if int(function_value['start']) < int(self.line) < int(
function_value['end']):
in_this_function = '<---- {0}'.format(self.line)
if block_position == 0:
block_start = function_value['start']
block_end = int(self.line) - 1
elif block_position == 1:
block_start = int(self.line)
block_end = int(function_value['end']) - 1
logging.debug("触发行所在函数: {0} ({1} - {2}) {3}".format(
function_name, function_value['start'],
function_value['end'], in_this_function))
else:
# 没有functions时,以触发行来分割整个文件
if block_position == 0:
block_start = 1
block_end = int(self.line) - 1
elif block_position == 1:
block_start = int(self.line) + 1
block_end = sum(1 for l in open(self.file_path))
logging.debug("没有找到任何方法,将以整个文件分割.")
# get param block code
line_rule = "{0},{1}p".format(block_start, block_end)
code = File(self.file_path).lines(line_rule)
logging.info('取出代码: {0} - {1}p'.format(block_start, block_end))
return code
def all(self):
projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all()
for project in projects:
payload = json.dumps({
"key": self.key,
"target": project.repository,
"branch": self.branch
})
try:
response = requests.post(self.api.format('add'), data=payload, headers=self.headers)
response_json = response.json()
logging.info(project.repository, response_json)
except (requests.ConnectionError, requests.HTTPError) as e:
logging.critical("API Add failed: {0}".format(e))
def run(self, is_all=None, pid=None):
if bool(is_all) is True:
message = '[START] Pull all projects code'
print(message)
logging.info(message)
projects = CobraProjects.query.with_entities(
CobraProjects.repository).filter(
CobraProjects.status == CobraProjects.get_status(
'on')).all()
for project in projects:
if '.git' not in project.repository:
continue
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(
msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
message = '[END] Scan all projects'
print(message)
logging.info(message)
elif pid is not None:
project = CobraProjects.query.filter_by(id=pid).first()
if project is None:
message = 'Project not found'
print(message)
logging.critical(message)
else:
if '.git' not in project.repository:
message = 'Not git repository'
print(message)
logging.info(message)
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(
msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
else:
message = 'Please set --target param'
print(message)
logging.critical(message)
sys.exit()
def __init__(self, time_type, month=None):
if time_type not in time_types:
logging.critical('Time type exception')
return
self.time_type_de = time_type_des[time_type]
# mail
mark = ''
if month is None:
c_month = int(datetime.datetime.today().strftime("%m"))
else:
c_month = int(month)
if time_type == 'w':
c_week = int(datetime.datetime.today().strftime("%U"))
mark = 'W{week}'.format(week=c_week)
elif time_type == 'm':
mark = 'M{month}'.format(month=c_month)
elif time_type == 'q':
c_quarter = 0
if c_month in [1, 2, 3]:
c_quarter = 1
elif c_month in [4, 5, 6]:
c_quarter = 2
elif c_month in [7, 8, 9]:
c_quarter = 3
elif c_month in [10, 11, 12]:
c_quarter = 4
mark = 'Q{quarter}'.format(quarter=c_quarter)
self.subject = '[Cobra] 代码安全{0}报({mark})'.format(self.time_type_de,
mark=mark)
self.user = Config('email', 'user').value
self.name = Config('email', 'name').value
self.to = Config('report', 'to').value
self.host = Config('email', 'host').value
self.port = Config('email', 'port').value
self.password = Config('email', 'password').value
self.param = [
phantomjs,
os.path.join(Config().project_directory, 'scheduler', 'report.js'),
Config().project_directory, time_type
]
if month is not None:
self.param.append(month)
def push_third_party_vulnerabilities(self, vulnerabilities_id):
"""
Pushed to a third-party vulnerability management platform
:param vulnerabilities_id:
:return:
"""
try:
status = Config('third_party_vulnerabilities', 'status').value
if int(status):
q = Queue(self.project_name,
self.third_party_vulnerabilities_name,
self.third_party_vulnerabilities_type,
self.file_path, self.line_number, self.code_content,
vulnerabilities_id)
q.push()
except Exception as e:
print(traceback.print_exc())
logging.critical(e.message)
def all(self):
projects = CobraProjects.query.with_entities(
CobraProjects.repository).filter(
CobraProjects.status == CobraProjects.get_status('on')).all()
for project in projects:
payload = json.dumps({
"key": self.key,
"target": project.repository,
"branch": self.branch
})
try:
response = requests.post(self.api.format('add'),
data=payload,
headers=self.headers)
response_json = response.json()
logging.info(project.repository, response_json)
except (requests.ConnectionError, requests.HTTPError) as e:
logging.critical("API Add failed: {0}".format(e))
def run(self, pid=None):
if pid is None:
logging.critical("Please set --pid param")
sys.exit()
# Project info
project_info = CobraProjects.query.filter_by(id=pid).first()
if project_info.repository[0] == '/':
project_directory = project_info.repository
else:
project_directory = Git(project_info.repository).repo_directory
# Third-party ID
vuln_all = CobraVuls.query.all()
vuln_all_d = {}
for vuln in vuln_all:
vuln_all_d[vuln.id] = vuln.third_v_id
# Not fixed vulnerabilities
result_all = db.session().query(CobraRules, CobraResults).join(
CobraResults, CobraResults.rule_id == CobraRules.id).filter(
CobraResults.project_id == pid, CobraResults.status < 2).all()
for index, (rule, result) in enumerate(result_all):
# Rule
result_info = {
'task_id': result.task_id,
'project_id': result.project_id,
'project_directory': project_directory,
'rule_id': result.rule_id,
'result_id': result.id,
'file_path': result.file,
'line_number': result.line,
'code_content': result.code,
'third_party_vulnerabilities_name': rule.description,
'third_party_vulnerabilities_type': vuln_all_d[rule.vul_id]
}
# White list
white_list = []
ws = CobraWhiteList.query.with_entities(
CobraWhiteList.path).filter_by(project_id=result.project_id,
rule_id=result.rule_id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
Core(result_info, rule, project_info.name, white_list).repair()
def run(self, is_all=None, pid=None):
if bool(is_all) is True:
message = '[START] Pull all projects code'
print(message)
logging.info(message)
projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all()
for project in projects:
if '.git' not in project.repository:
continue
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
message = '[END] Scan all projects'
print(message)
logging.info(message)
elif pid is not None:
project = CobraProjects.query.filter_by(id=pid).first()
if project is None:
message = 'Project not found'
print(message)
logging.critical(message)
else:
if '.git' not in project.repository:
message = 'Not git repository'
print(message)
logging.info(message)
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
else:
message = 'Please set --target param'
print(message)
logging.critical(message)
sys.exit()
def run(self, pid=None):
if pid is None:
logging.critical("Please set --pid param")
sys.exit()
# Project info
project_info = CobraProjects.query.filter_by(id=pid).first()
if project_info.repository[0] == '/':
project_directory = project_info.repository
else:
project_directory = Git(project_info.repository).repo_directory
# Third-party ID
vuln_all = CobraVuls.query.all()
vuln_all_d = {}
for vuln in vuln_all:
vuln_all_d[vuln.id] = vuln.third_v_id
# Not fixed vulnerabilities
result_all = db.session().query(CobraRules, CobraResults).join(CobraResults, CobraResults.rule_id == CobraRules.id).filter(
CobraResults.project_id == pid,
CobraResults.status < 2
).all()
for index, (rule, result) in enumerate(result_all):
# Rule
result_info = {
'task_id': result.task_id,
'project_id': result.project_id,
'project_directory': project_directory,
'rule_id': result.rule_id,
'result_id': result.id,
'file_path': result.file,
'line_number': result.line,
'code_content': result.code,
'third_party_vulnerabilities_name': rule.description,
'third_party_vulnerabilities_type': vuln_all_d[rule.vul_id]
}
# White list
white_list = []
ws = CobraWhiteList.query.with_entities(CobraWhiteList.path).filter_by(project_id=result.project_id, rule_id=result.rule_id, status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
Core(result_info, rule, project_info.name, white_list).repair()
def __init__(self, time_type, month=None):
if time_type not in time_types:
logging.critical('Time type exception')
return
self.time_type_de = time_type_des[time_type]
# mail
mark = ''
if month is None:
c_month = int(datetime.datetime.today().strftime("%m"))
else:
c_month = int(month)
if time_type == 'w':
c_week = int(datetime.datetime.today().strftime("%U"))
mark = 'W{week}'.format(week=c_week)
elif time_type == 'm':
mark = 'M{month}'.format(month=c_month)
elif time_type == 'q':
c_quarter = 0
if c_month in [1, 2, 3]:
c_quarter = 1
elif c_month in [4, 5, 6]:
c_quarter = 2
elif c_month in [7, 8, 9]:
c_quarter = 3
elif c_month in [10, 11, 12]:
c_quarter = 4
mark = 'Q{quarter}'.format(quarter=c_quarter)
self.subject = '[Cobra] 代码安全{0}报({mark})'.format(self.time_type_de, mark=mark)
self.user = Config('email', 'user').value
self.name = Config('email', 'name').value
self.to = Config('report', 'to').value
self.host = Config('email', 'host').value
self.port = Config('email', 'port').value
self.password = Config('email', 'password').value
self.param = [phantomjs, os.path.join(Config().project_directory, 'scheduler', 'report.js'), Config().project_directory, time_type]
if month is not None:
self.param.append(month)
def capture(self):
"""
Use PhantomJS to capture report page
:return: boolean
"""
capture = None
p = subprocess.Popen(self.param, stdout=subprocess.PIPE)
result, err = p.communicate()
if 'Critical' in result:
logging.critical('Capture exception')
return False
lines = result.split('\n')
for l in lines:
if 'reports' in l:
capture = l.split(':')[1].strip()
if capture is None:
logging.critical('get capture image file failed')
return False
else:
return os.path.join(Config().project_directory, capture)
def capture(self):
"""
Use PhantomJS to capture report page
:return: boolean
"""
capture = None
p = subprocess.Popen(self.param, stdout=subprocess.PIPE)
result, err = p.communicate()
if 'Critical' in result:
logging.critical('Capture exception')
return False
lines = result.split('\n')
for l in lines:
if 'reports' in l:
capture = l.split(':')[1].strip()
if capture is None:
logging.critical('get capture image file failed')
return False
else:
return os.path.join(Config().project_directory, capture)
def __init__(self, time_type):
if time_type not in time_types:
logging.critical('Time type exception')
return
self.time_type_de = time_type_des[time_type]
# mail
wd = int(datetime.datetime.today().strftime("%U"))
self.subject = '[Cobra] 代码安全{0}报(W{1})'.format(self.time_type_de, wd)
self.user = Config('email', 'user').value
self.name = Config('email', 'name').value
self.to = Config('report', 'to').value
self.host = Config('email', 'host').value
self.port = Config('email', 'port').value
self.password = Config('email', 'password').value
self.param = [
phantomjs,
os.path.join(Config().project_directory, 'scheduler', 'report.js'),
Config().project_directory, time_type
]
def repair(self):
"""
Scan vulnerabilities is repair
:flow:
- exist file [add]
- test file
- whitelist file
- special file
- annotation
- rule
:return: (Status, Result)
"""
self.method = 1
# Full path
self.file_path = self.project_directory + self.file_path
"""
When the targeting rule is empty or the line number is 0, it means that this type of language (all suffixes in that language) is counted as a vulnerability
Their repair method is only one: delete the file
"""
if self.rule_location == '' or self.line_number == 0:
logging.info("Find special files: RID{0}".format(self.rule_id))
# Check if the file exists
if os.path.isfile(self.file_path) is False:
# If the file is not found, the vulnerability state is fixed
logging.info("Deleted file repair is complete {0}".format(self.file_path))
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
else:
return
# Not exist file
if os.path.isfile(self.file_path) is False:
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
# Test file
if self.is_test_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_test_file
self.process_vulnerabilities()
return
"""
Cobra Skip
@cobra const
`@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]`
"""
file_content = File(self.file_path).read_file()
ret_regex_const = re.findall(r'@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]', file_content)
if len(ret_regex_const) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_const_file
self.process_vulnerabilities()
return
"""
@cobra third-party
`@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]`
"""
ret_regex_third_party = re.findall(r'@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]', file_content)
if len(ret_regex_third_party) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_third_party
self.process_vulnerabilities()
return
# Remove the trigger code (actual file)
trigger_code = File(self.file_path).lines("{0}p".format(self.line_number))
if trigger_code is False:
logging.critical("Failed to fetch the trigger code {0}".format(self.code_content))
self.status = self.status_fixed
self.repair_code = self.repair_code_empty_code
self.process_vulnerabilities()
return
self.code_content = trigger_code
# Whitelist
if self.is_white_list():
self.status = self.status_fixed
self.repair_code = self.repair_code_whitelist
self.process_vulnerabilities()
logging.info("In white list {0}".format(self.file_path))
return
# Special file
if self.is_special_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_special_file
self.process_vulnerabilities()
logging.info("Special File: {0}".format(self.file_path))
return
# Annotation
if self.is_annotation():
self.status = self.status_fixed
self.repair_code = self.repair_code_annotation
self.process_vulnerabilities()
logging.info("In Annotation {0}".format(self.code_content))
return
# Modify
ret_regex = re.findall(self.rule_location, trigger_code.strip())
if len(ret_regex) == 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_modify
self.process_vulnerabilities()
return
# Fixed
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location, self.file_path, self.line_number, self.code_content)
if parse_instance.is_repair(self.rule_repair, self.block_repair):
logging.info("Static: repaired")
# Fixed
self.status = self.status_fixed
self.repair_code = self.repair_code_fixed
self.process_vulnerabilities()
return
else:
logging.critical("[repair] not fixed")
return
except:
logging.info(traceback.print_exc())
return
def functions(self):
"""
获取该文件所有函数方法
:return:
"""
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
if 'darwin' == sys.platform:
ggrep = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename or 'grep' == filename:
ggrep = os.path.join(root, filename)
if ggrep == '':
logging.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if self.language not in self.regex:
logging.info("Undefined language's functions regex {0}".format(
self.language))
return False
regex_functions = self.regex[self.language]['functions']
param = [grep, "-n", "-r", "-P"] + [regex_functions, self.file_path]
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
if len(result[0]):
functions = {}
lines = str(result[0]).strip().split("\n")
prev_function_name = ''
for index, line in enumerate(lines):
line = line.strip()
if line == '':
logging.info('Empty')
continue
function = line.split(':')
if len(function) < 2:
logging.info("没有找到分隔符(:)")
regex_annotation = self.regex[self.language]['annotation']
string = re.findall(regex_annotation, function[1].strip())
if len(string) >= 1 and string[0] != '':
logging.info("该函数为注释行")
function_name = re.findall(regex_functions,
function[1].strip())
if len(function_name) == 1:
function_name = function_name[0]
if index > 0 and prev_function_name in functions:
functions[prev_function_name]['end'] = function[0]
prev_function_name = function_name
functions[function_name] = {
'start': function[0],
'end': None # next function's start
}
else:
logging.info("无法找到函数名: {0}".format(line))
end = sum(1 for l in open(self.file_path))
for name, value in functions.items():
if value['end'] is None:
functions[name]['end'] = end
return functions
else:
return False
def analyse(self):
if self.directory is None:
logging.critical("Please set directory")
sys.exit()
logging.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files(self.task_id)
logging.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
logging.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
logging.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
# `grep` (`ggrep` on Mac)
grep = '/bin/grep'
# `find` (`gfind` on Mac)
find = '/bin/find'
if 'darwin' == sys.platform:
ggrep = ''
gfind = ''
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename or 'grep' == filename:
ggrep = os.path.join(root, filename)
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/findutils'):
for filename in file_names:
if 'gfind' == filename:
gfind = os.path.join(root, filename)
if ggrep == '':
logging.critical("brew install ggrep pleases!")
sys.exit(0)
else:
grep = ggrep
if gfind == '':
logging.critical("brew install findutils pleases!")
sys.exit(0)
else:
find = gfind
"""
all vulnerabilities
vulnerabilities_all[vuln_id] = {'name': 'vuln_name', 'third_v_id': 'third_v_id'}
"""
vulnerabilities_all = {}
vulnerabilities = CobraVuls.query.all()
for v in vulnerabilities:
vulnerabilities_all[v.id] = {
'name': v.name,
'third_v_id': v.third_v_id
}
for rule in rules:
rule.regex_location = rule.regex_location.strip()
rule.regex_repair = rule.regex_repair.strip()
logging.info('------------------ RID: {0} {1} {2}'.format(
self.project_id, rule.id, rule.description))
# Filters
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
logging.critical("Rule Language Error")
sys.exit(0)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
if rule.regex_location == "":
filters = []
for index, e in enumerate(extensions):
if index > 1:
filters.append('-o')
filters.append('-name')
filters.append('*' + e)
# Find Special Ext Files
param = [find, self.directory, "-type", "f"] + filters
else:
filters = []
for e in extensions:
filters.append('--include=*' + e)
# explode dirs
explode_dirs = ['.svn', '.cvs', '.hg', '.git', '.bzr']
for explode_dir in explode_dirs:
filters.append('--exclude-dir={0}'.format(explode_dir))
# -s suppress error messages / -n Show Line number / -r Recursive / -P Perl regular expression
param = [grep, "-s", "-n", "-r", "-P"] + filters + [
rule.regex_location, self.directory
]
logging.debug(rule.regex_location)
p = subprocess.Popen(param, stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
lines = str(result[0]).strip().split("\n")
for line in lines:
line = line.strip()
if line == '':
continue
# 处理grep结果
if ':' in line:
line_split = line.split(':', 1)
file_path = line_split[0].strip()
code_content = line_split[1].split(':',
1)[1].strip()
line_number = line_split[1].split(':',
1)[0].strip()
else:
# 搜索文件
file_path = line
code_content = ''
line_number = 0
# 核心规则校验
result_info = {
'task_id':
self.task_id,
'project_id':
self.project_id,
'project_directory':
self.directory,
'rule_id':
rule.id,
'result_id':
None,
'file_path':
file_path,
'line_number':
line_number,
'code_content':
code_content,
'third_party_vulnerabilities_name':
vulnerabilities_all[rule.vul_id]['name'],
'third_party_vulnerabilities_type':
vulnerabilities_all[rule.vul_id]['third_v_id']
}
ret_status, ret_result = Core(result_info, rule,
self.project_name,
white_list).scan()
if ret_status is False:
logging.info("扫描 R: False {0}".format(ret_result))
continue
else:
logging.info('Not Found')
except Exception as e:
print(traceback.print_exc())
logging.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
logging.critical("Set start time failed:" + e.message)
logging.info("Scan Done")
def error_handler(self, uuid):
result = self.app.AsyncResult(uuid)
logging.critical('Task {0} raised exception: {1!r}\n{2!r}'.format(uuid, result.result, result.traceback))
def version(self, branch=None, new_version=None, old_version=None):
# Gitlab
if '.git' in self.target:
logging.info('Gitlab project')
# Git
if 'gitlab' in self.target:
username = config.Config('git', 'username').value
password = config.Config('git', 'password').value
else:
username = None
password = None
gg = git.Git(self.target,
branch=branch,
username=username,
password=password)
repo_author = gg.repo_author
repo_name = gg.repo_name
repo_directory = gg.repo_directory
# Git Clone Error
try:
clone_ret, clone_err = gg.clone()
if clone_ret is False:
return 4001, 'Clone Failed ({0})'.format(clone_err)
except NotExistError:
# update project status
p = CobraProjects.query.filter_by(
repository=self.target).first()
if p is not None:
if p.status == CobraProjects.get_status('on'):
p.status = CobraProjects.get_status('off')
db.session.add(p)
db.session.commit()
return 4001, 'Repository Does not exist!'
except AuthError:
logging.critical('Git Authentication Failed')
return 4001, 'Repository Authentication Failed'
elif 'svn' in self.target:
# SVN
repo_name = 'mogujie'
repo_author = 'all'
repo_directory = config.Config('upload', 'directory').value
else:
repo_name = 'Local Project'
repo_author = getpass.getuser()
repo_directory = self.target
if not os.path.exists(repo_directory):
return 1004, 'repo directory not exist ({0})'.format(
repo_directory)
if new_version == "" or old_version == "":
scan_way = 1
else:
scan_way = 2
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
# insert into task info table.
task = CobraTaskInfo(self.target, branch, scan_way, new_version,
old_version, 0, 0, 0, 1, 0, 0, current_time,
current_time)
p = CobraProjects.query.filter_by(repository=self.target).first()
project = None
# detection framework for project
framework, language = detection.Detection(repo_directory).framework()
if framework != '' or language != '':
project_framework = '{0} ({1})'.format(framework, language)
else:
project_framework = ''
project_id = 0
if not p:
# insert into project table.
project = CobraProjects(self.target, '', repo_name, repo_author,
project_framework, '', '', 1, current_time)
else:
project_id = p.id
# update project's framework
p.framework = project_framework
db.session.add(p)
try:
db.session.add(task)
if not p:
db.session.add(project)
db.session.commit()
if not p:
project_id = project.id
cobra_path = os.path.join(config.Config().project_directory,
'cobra.py')
if os.path.isfile(cobra_path) is not True:
return 1004, 'cobra.py not found'
# scan vulnerability
subprocess.Popen([
'python', cobra_path, "scan", "-p",
str(project_id), "-i",
str(task.id), "-t", repo_directory
])
# statistic code
subprocess.Popen([
'python', cobra_path, "statistic", "-i",
str(task.id), "-t", repo_directory
])
# check repair
subprocess.Popen(
['python', cobra_path, "repair", "-p",
str(project_id)])
result = dict()
result['scan_id'] = task.id
result['project_id'] = project_id
result['msg'] = u'success'
return 1001, result
except Exception as e:
return 1004, 'Unknown error, try again later?' + e.message
def error_handler(self, uuid):
result = self.app.AsyncResult(uuid)
logging.critical('Task {0} raised exception: {1!r}\n{2!r}'.format(
uuid, result.result, result.traceback))
def process_vulnerabilities(self):
"""
Process vulnerabilities
Write vulnerabilities / change the vulnerability status / push third-party vulnerability management platform
:return:
"""
# default status
if self.status is None:
self.status = self.status_init
# Handle relative paths
self.file_path = self.file_path.replace(self.project_directory, '')
# In scan
if self.method == 0:
"""
On Scan mode(method=1) only deal to new not fixed vulnerability(status=0)
"""
if self.status == 0:
# Check whether the unpatched vulnerability has been swept
# Line number 0 when the search for special documents (not to bring the line number)
if self.line_number == 0:
exist_result = CobraResults.query.filter(
CobraResults.project_id == self.project_id,
CobraResults.rule_id == self.rule_id,
CobraResults.file == self.file_path,
).order_by(CobraResults.id.desc()).first()
else:
exist_result = CobraResults.query.filter(
CobraResults.project_id == self.project_id,
CobraResults.rule_id == self.rule_id,
CobraResults.file == self.file_path,
CobraResults.line == self.line_number,
).order_by(CobraResults.id.desc()).first()
# Not fixed vulnerabilities have not been swept before, is written to the vulnerability database
if exist_result is None:
self.insert_vulnerabilities()
else:
logging.debug("This vulnerabilities already exist!")
elif self.method == 1:
"""
On Repair (method=1)
"""
if self.status == self.status_fixed:
# View the original status of the vulnerability
exist_result = CobraResults.query.filter_by(
id=self.result_id).first()
if exist_result is not None:
if exist_result.status < self.status_fixed:
# update vulnerability status is Fixed
exist_result.status = self.status_fixed
exist_result.repair = self.repair_code
db.session.add(exist_result)
db.session.commit()
else:
logging.critical("Vulnerabilities status not < fixed")
else:
logging.critical(
"Not found vulnerabilities on repair method")
else:
logging.critical("Core method not initialize")
def repair(self):
"""
Scan vulnerabilities is repair
:flow:
- exist file [add]
- test file
- whitelist file
- special file
- annotation
- rule
:return: (Status, Result)
"""
self.method = 1
# Full path
self.file_path = self.project_directory + self.file_path
"""
When the targeting rule is empty or the line number is 0, it means that this type of language (all suffixes in that language) is counted as a vulnerability
Their repair method is only one: delete the file
"""
if self.rule_location == '' or self.line_number == 0:
logging.info("Find special files: RID{0}".format(self.rule_id))
# Check if the file exists
if os.path.isfile(self.file_path) is False:
# If the file is not found, the vulnerability state is fixed
logging.info("Deleted file repair is complete {0}".format(
self.file_path))
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
else:
return
# Not exist file
if os.path.isfile(self.file_path) is False:
self.status = self.status_fixed
self.repair_code = self.repair_code_not_exist_file
self.process_vulnerabilities()
return
# Test file
if self.is_test_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_test_file
self.process_vulnerabilities()
return
"""
Cobra Skip
@cobra const
`@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]`
"""
file_content = File(self.file_path).read_file()
ret_regex_const = re.findall(
r'@[cC][oO][bB][rR][aA]\s*[cC][oO][nN][sS][tT]', file_content)
if len(ret_regex_const) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_const_file
self.process_vulnerabilities()
return
"""
@cobra third-party
`@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]`
"""
ret_regex_third_party = re.findall(
r'@[cC][oO][bB][rR][aA]\s*[tT][hH][iI][rR][dD]-[pP][aA][rR][tT][yY]',
file_content)
if len(ret_regex_third_party) > 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_third_party
self.process_vulnerabilities()
return
# Remove the trigger code (actual file)
trigger_code = File(self.file_path).lines("{0}p".format(
self.line_number))
if trigger_code is False:
logging.critical("Failed to fetch the trigger code {0}".format(
self.code_content))
self.status = self.status_fixed
self.repair_code = self.repair_code_empty_code
self.process_vulnerabilities()
return
self.code_content = trigger_code
# Whitelist
if self.is_white_list():
self.status = self.status_fixed
self.repair_code = self.repair_code_whitelist
self.process_vulnerabilities()
logging.info("In white list {0}".format(self.file_path))
return
# Special file
if self.is_special_file():
self.status = self.status_fixed
self.repair_code = self.repair_code_special_file
self.process_vulnerabilities()
logging.info("Special File: {0}".format(self.file_path))
return
# Annotation
if self.is_annotation():
self.status = self.status_fixed
self.repair_code = self.repair_code_annotation
self.process_vulnerabilities()
logging.info("In Annotation {0}".format(self.code_content))
return
# Modify
ret_regex = re.findall(self.rule_location, trigger_code.strip())
if len(ret_regex) == 0:
self.status = self.status_fixed
self.repair_code = self.repair_code_modify
self.process_vulnerabilities()
return
# Fixed
if self.is_can_parse() and self.rule_repair.strip() != '':
try:
parse_instance = parse.Parse(self.rule_location,
self.file_path, self.line_number,
self.code_content)
if parse_instance.is_repair(self.rule_repair,
self.block_repair):
logging.info("Static: repaired")
# Fixed
self.status = self.status_fixed
self.repair_code = self.repair_code_fixed
self.process_vulnerabilities()
return
else:
logging.critical("[repair] not fixed")
return
except:
logging.info(traceback.print_exc())
return
