def pull_code(self, branch='master'):
logging.info('Gitlab project')
# Git
if 'gitlab' in self.target:
username = config.Config('git', 'username').value
password = config.Config('git', 'password').value
else:
username = None
password = None
gg = git.Git(self.target, branch=branch, username=username, password=password)
# Git Clone Error
try:
clone_ret, clone_err = gg.clone()
if clone_ret is False:
return 4001, 'Clone Failed ({0})'.format(clone_err), gg
except NotExistError:
# update project status
p = CobraProjects.query.filter_by(repository=self.target).first()
if p is not None:
if p.status == CobraProjects.get_status('on'):
p.status = CobraProjects.get_status('off')
db.session.add(p)
db.session.commit()
return 4001, 'Repository Does not exist!', gg
except AuthError:
logging.critical('Git Authentication Failed')
return 4001, 'Repository Authentication Failed', gg
return 1001, 'Success', gg
def pull_code(self, branch='master'):
logging.info('Gitlab project')
# Git
if 'gitlab' in self.target:
username = config.Config('git', 'username').value
password = config.Config('git', 'password').value
else:
username = None
password = None
gg = git.Git(self.target,
branch=branch,
username=username,
password=password)
# Git Clone Error
try:
clone_ret, clone_err = gg.clone()
if clone_ret is False:
return 4001, 'Clone Failed ({0})'.format(clone_err), gg
except NotExistError:
# update project status
p = CobraProjects.query.filter_by(repository=self.target).first()
if p is not None:
if p.status == CobraProjects.get_status('on'):
p.status = CobraProjects.get_status('off')
db.session.add(p)
db.session.commit()
return 4001, 'Repository Does not exist!', gg
except AuthError:
logging.critical('Git Authentication Failed')
return 4001, 'Repository Authentication Failed', gg
return 1001, 'Success', gg
def compress(self):
dc = decompress.Decompress(self.target)
ret, result_d = dc.decompress()
if ret is False:
return 1002, result_d
else:
directory = result_d
logging.info("Scan directory: {0}".format(directory))
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
p = CobraProjects.query.filter_by(repository=directory).first()
# detection framework for project
framework, language = detection.Detection(directory).framework()
if framework != '' or language != '':
project_framework = '{0} ({1})'.format(framework, language)
else:
project_framework = ''
if not p:
# insert into project table.
repo_name = directory.split('/')[-1]
project = CobraProjects(directory, '', repo_name, 'Upload',
project_framework, '', '', 1, current_time)
db.session.add(project)
db.session.commit()
project_id = project.id
else:
project_id = p.id
# update project's framework
p.framework = project_framework
db.session.add(p)
task = CobraTaskInfo(directory, '', 3, '', '', 0, 0, 0, 1, 0, 0,
current_time, current_time)
db.session.add(task)
db.session.commit()
cobra_path = os.path.join(config.Config().project_directory,
'cobra.py')
if os.path.isfile(cobra_path) is not True:
return 1004, 'Cobra Not Found'
# 扫描漏洞
subprocess.Popen([
'python', cobra_path, "scan", "-p",
str(project_id), "-i",
str(task.id), "-t", directory
])
# 统计代码行数
subprocess.Popen([
'python', cobra_path, "statistic", "-i",
str(task.id), "-t", directory
])
# 检测漏洞修复状况
subprocess.Popen(
['python', cobra_path, "repair", "-p",
str(project_id)])
result = dict()
result['scan_id'] = task.id
result['project_id'] = project_id
result['msg'] = u'success'
return 1001, result
def run(self, is_all=None, pid=None):
if bool(is_all) is True:
message = '[START] Pull all projects code'
print(message)
logging.info(message)
projects = CobraProjects.query.with_entities(
CobraProjects.repository).filter(
CobraProjects.status == CobraProjects.get_status(
'on')).all()
for project in projects:
if '.git' not in project.repository:
continue
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(
msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
message = '[END] Scan all projects'
print(message)
logging.info(message)
elif pid is not None:
project = CobraProjects.query.filter_by(id=pid).first()
if project is None:
message = 'Project not found'
print(message)
logging.critical(message)
else:
if '.git' not in project.repository:
message = 'Not git repository'
print(message)
logging.info(message)
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(
msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
else:
message = 'Please set --target param'
print(message)
logging.critical(message)
sys.exit()
def all(self):
projects = CobraProjects.query.with_entities(
CobraProjects.repository).filter(
CobraProjects.status == CobraProjects.get_status('on')).all()
for project in projects:
payload = json.dumps({
"key": self.key,
"target": project.repository,
"branch": self.branch
})
try:
response = requests.post(self.api.format('add'),
data=payload,
headers=self.headers)
response_json = response.json()
logging.info(project.repository, response_json)
except (requests.ConnectionError, requests.HTTPError) as e:
logging.critical("API Add failed: {0}".format(e))
def add_project():
if not ValidateClass.check_login():
return redirect(ADMIN_URL + '/index')
if request.method == "POST":
vc = ValidateClass(request, "name", "repository", "url", "author", "pe", "remark")
ret, msg = vc.check_args()
if not ret:
return jsonify(tag="danger", msg=msg)
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
project = CobraProjects(vc.vars.repository, vc.vars.url, vc.vars.name, vc.vars.author, '', vc.vars.pe, vc.vars.remark, current_time)
try:
db.session.add(project)
db.session.commit()
return jsonify(tag='success', msg='save success.')
except:
return jsonify(tag='danger', msg='Unknown error.')
else:
return render_template('backend/project/add_project.html', data={})
def reports(vid, start_time, end_time):
projects = CobraProjects.query.order_by(CobraProjects.id.asc()).all()
rank = []
count_project_not_fixed = 0
count_project_fixed = 0
count_vulnerability_not_fixed = 0
count_vulnerability_fixed = 0
special_rules_ids = []
if vid is 0:
vulnerability_fixed_week = CobraResults.query.with_entities(
CobraResults.id).filter(
CobraResults.updated_at > '2016-11-28 00:00:00',
CobraResults.updated_at < '2016-11-04 23:59:59',
CobraResults.status == 2).count()
vulnerability_not_fixed_week = CobraResults.query.with_entities(
CobraResults.id).filter(
CobraResults.updated_at > '2016-11-28 00:00:00',
CobraResults.updated_at < '2016-11-04 23:59:59',
CobraResults.status < 2).count()
else:
rules = CobraRules.query.with_entities(
CobraRules.id).filter(CobraRules.vul_id == vid).all()
for rule in rules:
special_rules_ids.append(rule.id)
vulnerability_fixed_week = CobraResults.query.filter(
CobraResults.rule_id.in_(special_rules_ids),
CobraResults.created_at > '2016-11-28 00:00:00',
CobraResults.created_at < '2016-11-04 23:59:59',
CobraResults.status == 2).count()
vulnerability_not_fixed_week = CobraResults.query.with_entities(
CobraResults.id).filter(
CobraResults.updated_at > '2016-11-28 00:00:00',
CobraResults.updated_at < '2016-11-04 23:59:59',
CobraResults.status < 2).count()
filter_group = (
CobraResults.created_at > '{0} 00:00:00'.format(start_time),
CobraResults.created_at < '{0} 23:59:59'.format(end_time),
)
for project in projects:
if vid is 0:
filter_group_total_base = (CobraResults.project_id == project.id, )
if start_time == '0' and end_time == '0':
filter_group_total = filter_group_total_base + ()
count_total = CobraResults.query.filter(
*filter_group_total).count()
else:
filter_group_total = filter_group + filter_group_total_base + (
)
count_total = CobraResults.query.filter(
*filter_group_total).count()
else:
filter_group_total_base = (
CobraResults.project_id == project.id,
CobraResults.rule_id.in_(special_rules_ids),
)
if start_time == '0' and end_time == '0':
filter_group_total = filter_group_total_base + ()
count_total = CobraResults.query.filter(
*filter_group_total).count()
else:
filter_group_total = filter_group + filter_group_total_base + (
)
count_total = CobraResults.query.filter(
*filter_group_total).count()
# detect project Cobra configuration file
if project.repository[0] == '/':
project_directory = project.repository
else:
project_directory = Git(project.repository).repo_directory
cobra_properties = config.properties(
os.path.join(project_directory, 'cobra'))
need_scan = True
if 'scan' in cobra_properties:
need_scan = common.to_bool(cobra_properties['scan'])
if need_scan:
if vid is 0:
filter_group_fixed_base = (
CobraResults.project_id == project.id,
CobraResults.status == 2,
)
if start_time == '0' and end_time == '0':
filter_group_fixed = filter_group_fixed_base + ()
count_fixed = CobraResults.query.filter(
*filter_group_fixed).count()
else:
filter_group_fixed = filter_group + filter_group_fixed_base + (
)
count_fixed = CobraResults.query.filter(
*filter_group_fixed).count()
else:
filter_group_fixed_base = (
CobraResults.project_id == project.id,
CobraResults.status == 2,
CobraResults.rule_id.in_(special_rules_ids))
if start_time == '0' and end_time == '0':
filter_group_fixed = filter_group_fixed_base + ()
count_fixed = CobraResults.query.filter(
*filter_group_fixed).count()
else:
filter_group_fixed = filter_group + filter_group_fixed_base + (
)
count_fixed = CobraResults.query.filter(
*filter_group_fixed).count()
if project.status == 1:
count_not_fixed = count_total - count_fixed
remark = ''
else:
count_fixed = count_total
count_not_fixed = 0
remark = 'deleted'
else:
count_fixed = count_total
count_not_fixed = 0
remark = 'offline'
# update project status
if project.status == CobraProjects.get_status('on'):
project.status = CobraProjects.get_status('off')
db.session.add(project)
db.session.commit()
logging.info(
'Update project status (./cobra) {project}'.format(
project=project.repository))
if count_total != 0:
if need_scan:
if project.status == 1:
if count_not_fixed == 0:
count_project_fixed += 1
count_vulnerability_fixed += count_fixed
ret_whole = 'fixed'
else:
count_project_not_fixed += 1
count_vulnerability_fixed += count_fixed
count_vulnerability_not_fixed += count_not_fixed
ret_whole = 'not_fixed'
else:
# deleted project
count_project_fixed += 1
count_vulnerability_fixed += count_fixed
ret_whole = 'fixed'
else:
count_project_fixed += 1
count_vulnerability_fixed += count_fixed
ret_whole = 'fixed'
report = 'http://' + config.Config(
'cobra', 'domain').value + '/report/' + str(project.id)
s = {
'name': project.name,
'id': project.id,
'not_fixed': count_not_fixed,
'fixed': count_fixed,
'total': count_total,
'remark': remark,
'author': project.author,
'report': report,
'class': ret_whole
}
rank.append(s)
rank = sorted(rank, key=lambda x: x['not_fixed'], reverse=True)
vulnerabilities_types = CobraVuls.query.all()
if start_time == '0':
start_time = ''
if end_time == '0':
end_time = ''
data = {
'rank': rank,
'vulnerabilities_types': vulnerabilities_types,
'vid': vid,
'count': {
'vulnerability': {
'not_fixed': count_vulnerability_not_fixed,
'fixed': count_vulnerability_fixed,
'total':
count_vulnerability_not_fixed + count_vulnerability_fixed
},
'project': {
'not_fixed': count_project_not_fixed,
'fixed': count_project_fixed,
'total': count_project_not_fixed + count_project_fixed
},
'week': {
'fixed':
"{0}({1})".format(
vulnerability_fixed_week,
common.percent(vulnerability_fixed_week,
count_vulnerability_fixed)),
'not_fixed':
"{0}({1})".format(
vulnerability_not_fixed_week,
common.percent(vulnerability_not_fixed_week,
count_vulnerability_not_fixed))
}
},
'filter': {
'start': start_time,
'end': end_time
}
}
return render_template("backend/report/report.html", data=data)
def version(self, branch=None, new_version=None, old_version=None):
# Gitlab
if '.git' in self.target:
logging.info('Gitlab project')
# Git
if 'gitlab' in self.target:
username = config.Config('git', 'username').value
password = config.Config('git', 'password').value
else:
username = None
password = None
gg = git.Git(self.target, branch=branch, username=username, password=password)
repo_author = gg.repo_author
repo_name = gg.repo_name
repo_directory = gg.repo_directory
# Git Clone Error
clone_ret, clone_err = gg.clone()
if clone_ret is False:
return 4001, 'Clone Failed ({0})'.format(clone_err)
elif 'svn' in self.target:
# SVN
repo_name = 'mogujie'
repo_author = 'all'
repo_directory = config.Config('upload', 'directory').value
else:
repo_name = 'Local Project'
repo_author = getpass.getuser()
repo_directory = self.target
if not os.path.exists(repo_directory):
return 1004, 'repo directory not exist ({0})'.format(repo_directory)
if new_version == "" or old_version == "":
scan_way = 1
else:
scan_way = 2
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
# insert into task info table.
task = CobraTaskInfo(self.target, branch, scan_way, new_version, old_version, 0, 0, 0, 1, 0, 0, current_time, current_time)
p = CobraProjects.query.filter_by(repository=self.target).first()
project = None
# detection framework for project
framework, language = detection.Detection(repo_directory).framework()
if framework != '' or language != '':
project_framework = '{0} ({1})'.format(framework, language)
else:
project_framework = ''
project_id = 0
if not p:
# insert into project table.
project = CobraProjects(self.target, '', repo_name, repo_author, project_framework, '', '', 1, current_time)
else:
project_id = p.id
# update project's framework
p.framework = project_framework
db.session.add(p)
try:
db.session.add(task)
if not p:
db.session.add(project)
db.session.commit()
if not p:
project_id = project.id
cobra_path = os.path.join(config.Config().project_directory, 'cobra.py')
if os.path.isfile(cobra_path) is not True:
return 1004, 'cobra.py not found'
# scan vulnerability
subprocess.Popen(['python', cobra_path, "scan", "-p", str(project_id), "-i", str(task.id), "-t", repo_directory])
# statistic code
subprocess.Popen(['python', cobra_path, "statistic", "-i", str(task.id), "-t", repo_directory])
# check repair
subprocess.Popen(['python', cobra_path, "repair", "-p", str(project_id)])
result = dict()
result['scan_id'] = task.id
result['project_id'] = project_id
result['msg'] = u'success'
return 1001, result
except Exception as e:
return 1004, 'Unknown error, try again later?' + e.message
def run(self, is_all=None, pid=None):
if bool(is_all) is True:
message = '[START] Pull all projects code'
print(message)
logging.info(message)
projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all()
for project in projects:
if '.git' not in project.repository:
continue
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
message = '[END] Scan all projects'
print(message)
logging.info(message)
elif pid is not None:
project = CobraProjects.query.filter_by(id=pid).first()
if project is None:
message = 'Project not found'
print(message)
logging.critical(message)
else:
if '.git' not in project.repository:
message = 'Not git repository'
print(message)
logging.info(message)
code, msg, gg = scan.Scan(project.repository).pull_code()
message = 'Pull code: {msg} {directory}'.format(msg=msg, directory=gg.repo_directory)
if code == 1001:
logging.info(message)
else:
logging.warning(message)
print(message)
else:
message = 'Please set --target param'
print(message)
logging.critical(message)
sys.exit()
def reports(vid, start_time, end_time):
projects = CobraProjects.query.order_by(CobraProjects.id.asc()).all()
rank = []
count_project_not_fixed = 0
count_project_fixed = 0
count_vulnerability_not_fixed = 0
count_vulnerability_fixed = 0
special_rules_ids = []
if vid is 0:
vulnerability_fixed_week = CobraResults.query.with_entities(CobraResults.id).filter(CobraResults.updated_at > '2016-11-28 00:00:00', CobraResults.updated_at < '2016-11-04 23:59:59', CobraResults.status == 2).count()
vulnerability_not_fixed_week = CobraResults.query.with_entities(CobraResults.id).filter(CobraResults.updated_at > '2016-11-28 00:00:00', CobraResults.updated_at < '2016-11-04 23:59:59', CobraResults.status < 2).count()
else:
rules = CobraRules.query.with_entities(CobraRules.id).filter(CobraRules.vul_id == vid).all()
for rule in rules:
special_rules_ids.append(rule.id)
vulnerability_fixed_week = CobraResults.query.filter(CobraResults.rule_id.in_(special_rules_ids), CobraResults.created_at > '2016-11-28 00:00:00', CobraResults.created_at < '2016-11-04 23:59:59', CobraResults.status == 2).count()
vulnerability_not_fixed_week = CobraResults.query.with_entities(CobraResults.id).filter(CobraResults.updated_at > '2016-11-28 00:00:00', CobraResults.updated_at < '2016-11-04 23:59:59', CobraResults.status < 2).count()
filter_group = (CobraResults.created_at > '{0} 00:00:00'.format(start_time), CobraResults.created_at < '{0} 23:59:59'.format(end_time),)
for project in projects:
if vid is 0:
filter_group_total_base = (CobraResults.project_id == project.id,)
if start_time == '0' and end_time == '0':
filter_group_total = filter_group_total_base + ()
count_total = CobraResults.query.filter(*filter_group_total).count()
else:
filter_group_total = filter_group + filter_group_total_base + ()
count_total = CobraResults.query.filter(*filter_group_total).count()
else:
filter_group_total_base = (CobraResults.project_id == project.id, CobraResults.rule_id.in_(special_rules_ids),)
if start_time == '0' and end_time == '0':
filter_group_total = filter_group_total_base + ()
count_total = CobraResults.query.filter(*filter_group_total).count()
else:
filter_group_total = filter_group + filter_group_total_base + ()
count_total = CobraResults.query.filter(*filter_group_total).count()
# detect project Cobra configuration file
if project.repository[0] == '/':
project_directory = project.repository
else:
project_directory = Git(project.repository).repo_directory
cobra_properties = config.properties(os.path.join(project_directory, 'cobra'))
need_scan = True
if 'scan' in cobra_properties:
need_scan = common.to_bool(cobra_properties['scan'])
if need_scan:
if vid is 0:
filter_group_fixed_base = (CobraResults.project_id == project.id, CobraResults.status == 2,)
if start_time == '0' and end_time == '0':
filter_group_fixed = filter_group_fixed_base + ()
count_fixed = CobraResults.query.filter(*filter_group_fixed).count()
else:
filter_group_fixed = filter_group + filter_group_fixed_base + ()
count_fixed = CobraResults.query.filter(*filter_group_fixed).count()
else:
filter_group_fixed_base = (CobraResults.project_id == project.id, CobraResults.status == 2, CobraResults.rule_id.in_(special_rules_ids))
if start_time == '0' and end_time == '0':
filter_group_fixed = filter_group_fixed_base + ()
count_fixed = CobraResults.query.filter(*filter_group_fixed).count()
else:
filter_group_fixed = filter_group + filter_group_fixed_base + ()
count_fixed = CobraResults.query.filter(*filter_group_fixed).count()
if project.status == 1:
count_not_fixed = count_total - count_fixed
remark = ''
else:
count_fixed = count_total
count_not_fixed = 0
remark = 'deleted'
else:
count_fixed = count_total
count_not_fixed = 0
remark = 'offline'
# update project status
if project.status == CobraProjects.get_status('on'):
project.status = CobraProjects.get_status('off')
db.session.add(project)
db.session.commit()
logging.info('Update project status (./cobra) {project}'.format(project=project.repository))
if count_total != 0:
if need_scan:
if project.status == 1:
if count_not_fixed == 0:
count_project_fixed += 1
count_vulnerability_fixed += count_fixed
ret_whole = 'fixed'
else:
count_project_not_fixed += 1
count_vulnerability_fixed += count_fixed
count_vulnerability_not_fixed += count_not_fixed
ret_whole = 'not_fixed'
else:
# deleted project
count_project_fixed += 1
count_vulnerability_fixed += count_fixed
ret_whole = 'fixed'
else:
count_project_fixed += 1
count_vulnerability_fixed += count_fixed
ret_whole = 'fixed'
report = 'http://' + config.Config('cobra', 'domain').value + '/report/' + str(project.id)
s = {
'name': project.name,
'id': project.id,
'not_fixed': count_not_fixed,
'fixed': count_fixed,
'total': count_total,
'remark': remark,
'author': project.author,
'report': report,
'class': ret_whole
}
rank.append(s)
rank = sorted(rank, key=lambda x: x['not_fixed'], reverse=True)
vulnerabilities_types = CobraVuls.query.all()
if start_time == '0':
start_time = ''
if end_time == '0':
end_time = ''
data = {
'rank': rank,
'vulnerabilities_types': vulnerabilities_types,
'vid': vid,
'count': {
'vulnerability': {
'not_fixed': count_vulnerability_not_fixed,
'fixed': count_vulnerability_fixed,
'total': count_vulnerability_not_fixed + count_vulnerability_fixed
},
'project': {
'not_fixed': count_project_not_fixed,
'fixed': count_project_fixed,
'total': count_project_not_fixed + count_project_fixed
},
'week': {
'fixed': "{0}({1})".format(vulnerability_fixed_week, common.percent(vulnerability_fixed_week, count_vulnerability_fixed)),
'not_fixed': "{0}({1})".format(vulnerability_not_fixed_week, common.percent(vulnerability_not_fixed_week, count_vulnerability_not_fixed))
}
},
'filter': {
'start': start_time,
'end': end_time
}
}
return render_template("backend/report/report.html", data=data)
def all(self):
projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all()
for project in projects:
payload = json.dumps({
"key": self.key,
"target": project.repository,
"branch": self.branch
})
try:
response = requests.post(self.api.format('add'), data=payload, headers=self.headers)
response_json = response.json()
logging.info(project.repository, response_json)
except (requests.ConnectionError, requests.HTTPError) as e:
logging.critical("API Add failed: {0}".format(e))