def batch(request):
"""
:param request:
:return:
"""
action = request.POST.get('action', '')
ids = request.POST.get('ids', '')
try:
if action == 'del':
group_list = GroupInfo.objects.filter(
id__in=[i for i in ids.split(',') if i])
for group in group_list:
obj = ProjectInfo.objects.filter(group__id=group.id).first()
if obj:
raise Exception("'{0}' 分类下存在项目, 请删除项目后在删除该分组。".format(
group.name))
group.delete()
return JsonResponse({"status": "ok"}, safe=False)
except Exception as ex:
import traceback
traceback.print_exc()
create_syslog_obj(title='删除项目分组失败',
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.ERROR)
return JsonResponse({"status": "failed", 'msg': str(ex)}, safe=False)
def start(*args, **kwargs):
"""
:param args:
:param kwargs:
:return:
"""
try:
logger.info('开始同步 GitLab 项目...')
start_time = time.time()
gitlab = GitLabOperator()
gitlab.add_projects()
end_time = time.time() - start_time
create_syslog_obj(title='GitLab 项目同步成功',
description='共计耗时:{0} 分钟'.format(
round(end_time / 60, 2)),
level=LOG_LEVEL.NOTIFICATION)
return True
except Exception as ex:
traceback.print_exc()
create_syslog_obj(
title='GitLab 同步失败',
description=ex,
stack_trace=traceback.format_exc(),
level=1,
)
return False
def __upload_upgrade(local_file, remote_file):
"""
:param local_file:
:param remote_file:
:return:
"""
try:
web_conf = get_config()
if web_conf['project']['upload_type'] == 2:
ftp = FTPWork(
host=web_conf['project']['ftp_host'],
port=web_conf['project']['ftp_port'],
username=web_conf['project']['ftp_username'],
password=web_conf['project']['ftp_password'],
debug=False
)
upgrade_path = os.path.join(web_conf['project']['ftp_path'], 'upgrade')
ftp.make_dir(upgrade_path)
remote_file = os.path.join(upgrade_path, remote_file)
for i in range(0, 3):
try:
ftp.upload_file(local_file, remote_file)
size = ftp.get_size(remote_file)
if size:
break
except Exception as ex:
if i == 2:
raise ex
ftp.close()
create_syslog_obj(
title='上传升级包成功',
description=remote_file,
level=LOG_LEVEL.INFO
)
return 'ftp://{0}:{1}{2}'.format(
web_conf['project']['ftp_host'],
web_conf['project']['ftp_port'],
os.path.join(upgrade_path, remote_file)
)
except Exception as ex:
import traceback
create_syslog_obj(
title='上传升级包失败',
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.ERROR
)
return None
def _add_members_by_project(self, project_git_id, project_name):
"""
更新会员信息
:param project_git_id:
:param project_name:
:return:
"""
members = {}
for member in self.api.get_members_by_project(project_id=project_git_id):
if not member:
continue
logger.info('[MEMBER] username: {0}, git id:{1}'.format(member['username'], member['id']))
try:
email = None
if 'email' in member:
email = member['email']
m = get_member_by_gitid(member['id'])
if not m or m.state != 'active':
member_obj = create_or_update(
git_id=member['id'],
name=member['name'],
username=member['username'],
state=member['state'],
web_url=member['web_url'],
email=email,
)
members[member['username']] = {
'json': member,
'obj': member_obj,
}
except Exception as ex:
import traceback
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目成员失败'.format(project_name),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
return members
def post(self, request, issue_id):
"""
:param request:
:param issue_id:
:return:
"""
try:
if 'application/json' not in request.content_type:
raise Exception(
u'"Content-type" 格式必须为 json 格式, 当前格式: {0}'.format(
request.content_type))
scm_id = request.data.get("scm_id", None)
scm_status = request.data.get("scm_status", None)
scm_url = request.data.get("scm_url", None)
update_issue_obj(
issue_id=issue_id,
scm_id=scm_id,
scm_status=scm_status,
scm_url=scm_url,
)
return JsonResponse(data={'issue_id': issue_id},
desc='更新成功',
status=status.HTTP_200_OK,
code=1)
except Exception as ex:
import traceback
traceback.print_exc()
create_syslog_obj(title='[IssueCallbackSet] 调用接口发生错误',
description=str(ex),
stack_trace=traceback.format_exc(),
sys_type=1,
level=LOG_LEVEL.CRITICAL)
return JsonResponse(desc=str(ex),
code=status.HTTP_400_BAD_REQUEST,
status=status.HTTP_200_OK)
def download_cloud_result(local_file, remote_file_log):
"""
:param local_file:
:param remote_file_log:
:return:
"""
try:
web_conf = get_config()
if web_conf['project']['upload_type'] == 2:
if not conf.storage['ftp']:
conf.storage['ftp'] = FTPWork(
host=web_conf['project']['ftp_host'],
port=web_conf['project']['ftp_port'],
username=web_conf['project']['ftp_username'],
password=web_conf['project']['ftp_password'],
)
conf.storage['ftp'].download_file(local_file, remote_file_log)
except Exception as ex:
import traceback
create_syslog_obj(title='下载扫描日志失败',
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.ERROR)
def sync_project_branch(self, project_id):
"""
更新项目的分支与tag
:return:
"""
project_obj = get_project_by_gitid(git_id=project_id)
try:
new_result = []
update_result = []
for branch in self.api.get_branches(project_id=project_id):
if not branch:
continue
try:
bran = get_repository_by_name(name=branch['name'], project_id=project_obj.id)
if 'commit' in branch:
last_commit_id = branch['commit']['id']
last_short_id = branch['commit']['short_id']
last_author_email = branch['commit']['author_email']
last_author_name = branch['commit']['author_name']
last_title = branch['commit']['title'],
else:
last_commit_id = None
last_short_id = None
last_author_email = None
last_author_name = None
last_title = None
if bran:
if bran.last_commit_id != branch['commit']['id']:
update_repository_obj(
repo_id=bran.id,
name=branch['name'],
merged=branch['merged'],
protected=branch['protected'],
developers_can_push=branch['developers_can_push'],
developers_can_merge=branch['developers_can_merge'],
last_commit_id=last_commit_id,
last_short_id=last_short_id,
last_author_email=last_author_email,
last_author_name=last_author_name,
last_title=last_title,
project_obj=project_obj,
)
update_result.append(branch['name'])
logger.info('[BRANCH] update branch name:{0}, project name:{0}'.format(
branch['name'],
project_obj.name)
)
else:
new_result.append(branch['name'])
create_repository_obj(
name=branch['name'],
merged=branch['merged'],
protected=branch['protected'],
developers_can_push=branch['developers_can_push'],
developers_can_merge=branch['developers_can_merge'],
last_commit_id=last_commit_id,
last_short_id=last_short_id,
last_author_email=last_author_email,
last_author_name=last_author_name,
last_title=last_title,
project_obj=project_obj,
)
logger.info('[BRANCH] new branch name:{0}, project name:{0}'.format(
branch['name'],
project_obj.name)
)
except Exception as ex:
import traceback
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目分支失败'.format(project_obj.name),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
if update_result:
create_pro_history_obj(
project_obj=project_obj,
title='更新分支 {0} 个'.format(len(update_result)),
description='更新：{0} 分支'.format('、'.join(update_result)),
type=icon_type.CODE_FORK
)
if new_result:
create_pro_history_obj(
project_obj=project_obj,
title='新增分支 {0} 个'.format(len(new_result)),
description='新增：{0} 分支'.format('、'.join(new_result)),
type=icon_type.TAGS
)
except Exception as ex:
logger.error(ex)
def add_projects(self, enable_sync_branch=True):
"""
添加项目
:return:
"""
logger.info('Start syncing project information...')
today = datetime.datetime.now()
for project in self.api.get_projects():
last_activity_at = utc2local(project['last_activity_at']) + datetime.timedelta(
days=self.activity_limit_month * 30)
if last_activity_at < today:
logger.warn('[SKIP] “{0}” 项目已超过 {1} 个月未活动, 跳过当前项目的信息同步, 最后活动时间: {2}。'
''.format(project['name'], self.activity_limit_month, project['last_activity_at']))
continue
logger.info('Start syncing "{0}" project member information ...'.format(project['name']))
try:
# member
members = self._add_members_by_project(project['id'], project['name'])
# group
group_obj = get_group_by_gitid(git_id=project['namespace']['id'])
if not group_obj:
logger.debug('"{0}" group not found.'.format(project['namespace']))
group_obj = create_group_obj(
git_id=project['namespace']['id'],
name=project['namespace']['name'],
parent_id=project['namespace']['parent_id'],
path=project['namespace']['path'],
full_path=project['namespace']['full_path'],
)
group_json = self.api.get_group_info(group_id=project['namespace']['id'])
if group_json:
group_obj = update_group_obj(
git_id=group_json['id'],
name=group_json['name'],
path=group_json['path'],
description=group_json['description'],
web_url=group_json['web_url'],
full_name=group_json['full_name'],
full_path=group_json['full_path'],
visibility_level=group_json['visibility_level'],
)
self._add_groups_members(group_obj)
# project
pro = get_project_by_gitid(project['id'])
i_ok = False
if pro:
last_activity_at = utc2local(project['last_activity_at']).strftime("%Y%m%d%H%M%S")
pro_last_activity_at = utc2local(pro.git_last_activity_at.astimezone(self.tz).strftime("%Y-%m-%d %H:%M:%S")).strftime("%Y%m%d%H%M%S")
# assert last_activity_at == pro_last_activity_at
# assert project['name'].lower() == pro.name.lower()
# assert project['ssh_url_to_repo'].lower() == pro.ssh_url_to_repo.lower()
if last_activity_at != pro_last_activity_at or \
project['name'].lower() != pro.name.lower() or \
project['ssh_url_to_repo'].lower() != pro.ssh_url_to_repo.lower():
i_ok = True
else:
i_ok = True
if not project['default_branch'] and pro:
i_ok = False
logger.warning('"{0}" is an empty project.'.format(project['web_url']))
if not i_ok:
logger.warning('[SKIP] [*] The project has not been changed, skip the update.')
continue
_name, _username, department = 'id: {0}'.format(project['creator_id']), '', None
project_obj = project_create_or_update(
group_obj=group_obj,
git_id=project['id'],
git_created_at=project['created_at'],
git_last_activity_at=project['last_activity_at'],
issues_enabled=project['issues_enabled'],
ssh_url_to_repo=project['ssh_url_to_repo'],
http_url_to_repo=project['http_url_to_repo'],
web_url=project['web_url'],
default_branch=project['default_branch'],
name=project['name'],
path=project['path'],
path_with_namespace=project['path_with_namespace'],
creator_id=project['creator_id'],
description=project['description'],
star_count=project['star_count'],
forks_count=project['forks_count'],
open_issues_count=0,
visibility_level=project['visibility_level'],
)
if pro:
title = '更新『{0}』项目'.format(project['name'])
description = '更新『{0}』项目成功，默认分支：{1}'.format(
project['name'], project['default_branch'] or '-'
)
create_pro_history_obj(
project_obj=pro,
title=title,
description=description,
type=icon_type.INFO
)
else:
title = '创建 『{0}』项目'.format(project['name'])
description = '创建『{0}』项目成功\n所属分组: {1}\n创建者：{2}({3})\n默认分支：{4}'.format(
project['name'], project['namespace']['name'],
_name, _username, project['default_branch'] or '-'
)
create_pro_history_obj(
project_obj=project_obj,
title=title,
description=description,
is_first=True,
type=icon_type.DATABASE
)
logger.debug('[PROJECT] project name:"{0}", git id: {1}, group name:"{2}".'.format(
project['name'],
project['id'],
group_obj.name))
# members
new_list = []
r_list = []
if not project_obj:
project_obj = pro
for username, item in members.items():
if item['obj'].state == 'active': # 有效账户
perm = get_pro_member_perm(member_id=item['obj'].id, project_id=project_obj.id)
if not perm:
create_pro_member_perm_obj(
project_obj=project_obj,
member_obj=item['obj'],
access_level=item['json']['access_level'],
expires_at=item['json']['expires_at']
)
if not project_obj.members.filter(id=item['obj'].id).exists():
new_list.append(username)
r_list.append(item['obj'])
if r_list:
if project_obj:
project_obj.members.add(*r_list)
if new_list:
create_pro_history_obj(
project_obj=project_obj,
title='发现新授权员工 {0} 个'.format(len(new_list)),
description='发现新增：{0} 等账号'.format('、'.join([_ for _ in new_list])),
type=icon_type.USER
)
# branch
if enable_sync_branch:
self.sync_project_branch(project_id=project['id'])
except Exception as ex:
import traceback
traceback.print_exc()
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目信息失败'.format(project['name']),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
def post(self, request):
"""
:param request:
:return:
"""
try:
if 'application/json' not in request.content_type:
raise Exception(
u'"Content-type" 格式必须为 json 格式, 当前格式: {0}'.format(
request.content_type))
secret = request.data.get("secret", None)
title = request.data.get("title", None)
description = request.data.get("description", None)
stack_trace = request.data.get("stack_trace", None)
module_id = request.data.get("module_id", None)
object_id = request.data.get("object_id", None)
ipv4 = request.data.get("ipv4", None)
sys_type = request.data.get("sys_type", 1)
level = request.data.get("level", LOG_LEVEL.INFO)
if secret:
decrypt_str = RSA.decrypt_str(secret)
json_data = ast.literal_eval(decrypt_str.decode('utf-8'))
if 'title' in json_data:
title = json_data['title']
if 'description' in json_data:
description = json_data['description']
if 'stack_trace' in json_data:
stack_trace = json_data['stack_trace']
if 'sys_type' in json_data:
sys_type = json_data['sys_type']
if 'level' in json_data:
level = json_data['level']
module_id = request.data.get("module_id", None)
object_id = request.data.get("object_id", None)
if not all((title, description)):
raise SeeCodeMissingImportantParameters(
"Parameter 'title, description' content is invalid.")
if not ipv4:
if 'HTTP_X_FORWARDED_FOR' in request.META:
ipv4 = request.META['HTTP_X_FORWARDED_FOR']
else:
ipv4 = request.META['REMOTE_ADDR']
create_syslog_obj(
title=title,
description=description,
stack_trace=stack_trace,
module_id=module_id,
object_id=object_id,
ipv4=ipv4,
sys_type=sys_type,
level=level,
)
return JsonResponse(data={},
desc='更新成功',
status=status.HTTP_200_OK,
code=1)
except Exception as ex:
import traceback
traceback.print_exc()
return JsonResponse(desc=str(ex),
code=status.HTTP_400_BAD_REQUEST,
status=status.HTTP_200_OK)
def dispatch_task(**kwargs):
"""
派发扫描任务服务
:param kwargs:
:return:
"""
task_obj = None
try:
task_group_id = kwargs.get('task_group_id', None)
task_type = kwargs.get('task_type', None)
group_id = kwargs.get('group_id', None)
app_id = kwargs.get('app_id', None)
branch = kwargs.get('branch', '')
is_force_scan = kwargs.get('is_force_scan', False)
scan_way = kwargs.get('scan_way', 1)
if not all((task_group_id, )):
raise Exception("任务分组不能为空!")
if task_type == tasktype.SINGLE: # 单项目
check_scan_task(app_id=app_id)
task_obj = create_scan_task(
task_group_id=task_group_id,
app_id=app_id,
is_force_scan=is_force_scan,
scan_way=scan_way,
)
send_task(task_id=task_obj.id)
else:
if task_type == tasktype.MULTIPLE: # 指定项目组
if not group_id:
raise Exception("分组未找到，ID:{0}．".format(group_id))
project_list = get_project_list_by_group_id(group_id=group_id)
else: # 所有项目组
project_list = ProjectInfo.objects.all()
for item in project_list:
try:
app_obj = get_app_by_branch(name=branch,
project_id=item.id)
if app_obj:
check_scan_task(app_id=app_obj.id)
task_obj = create_scan_task(
task_group_id=task_group_id,
app_id=app_obj.id,
is_force_scan=is_force_scan,
scan_way=scan_way,
)
send_task(task_id=task_obj.id)
else:
raise ApplicationNotFound(
'未找到 branch name:{0}, project id: {1} 对应的应用。'.
format(branch, item.id))
except (ScanTaskAlreadyExists, ApplicationNotFound) as ex:
import traceback
traceback.print_exc()
create_syslog_obj(
title='创建扫描任务失败',
description=str(ex),
stack_trace=traceback.format_exc(),
level=2,
is_read=False,
)
return True
except (ScanTaskAlreadyExists, ApplicationNotFound) as ex:
import traceback
traceback.print_exc()
create_syslog_obj(
title='创建扫描任务失败',
description=str(ex),
stack_trace=traceback.format_exc(),
level=2,
is_read=False,
)
return False
except Exception as ex:
import traceback
traceback.print_exc()
if task_obj:
update_task_failed(task_id=task_obj.id,
title=str(ex),
reason=traceback.format_exc())
create_syslog_obj(
title='创建扫描任务失败',
description=str(ex),
stack_trace=traceback.format_exc(),
level=1,
is_read=False,
)
return False