def batch_local_add(self, request, pk=None):
user = request.user
if not user.is_superuser:
return JsonResponse(R.build(msg="权限不足"))
image_name_str = request.POST.get("image_names", "")
image_names = image_name_str.split(",")
rsp_msg = []
for image_name in image_names:
if not image_name:
continue
if ":" not in image_name:
image_name += ":latest"
image_info = ImageInfo.objects.filter(
image_name=image_name).first()
if not image_info:
image_vul_name = image_name[:image_name.rfind(":")]
image_info = ImageInfo(image_name=image_name,
image_vul_name=image_vul_name,
image_desc=image_vul_name,
rank=2.5,
is_ok=False,
create_date=timezone.now(),
update_date=timezone.now())
image_info.save()
task_id = tasks.create_image_task(
image_info=image_info,
user_info=user,
request_ip=get_request_ip(request),
image_file=None)
if task_id:
rsp_msg.append("拉取镜像%s任务下发成功" % (image_name, ))
return JsonResponse(R.ok(data=rsp_msg))
def create(self, request, *args, **kwargs):
"""
创建镜像
:param request:
:param args:
:param kwargs:
:return:
"""
image_name = request.data.get("image_name", "")
image_vul_name = request.data.get("image_vul_name", "")
image_desc = request.data.get("image_desc", "")
image_port = request.data.get("image_port", "")
try:
image_rank = request.data.get("rank", default=2.5)
image_rank = float(image_rank)
except:
image_rank = 2.5
image_file = request.FILES.get("file")
image_info = None
if image_name:
if ":" not in image_name:
image_name += ":latest"
image_info = ImageInfo.objects.filter(image_name=image_name).first()
if not image_info:
image_vul_name = re.sub(r'[^a-z\-0-9]', "-", image_vul_name, flags=re.I).lower()
image_info = ImageInfo(image_name=image_name, image_vul_name=image_vul_name, image_desc=image_desc,
image_port=image_port, rank=image_rank, is_ok=False, create_date=timezone.now(),
update_date=timezone.now())
if not image_file:
image_info.save()
return JsonResponse(R.ok(data=image_info.image_id, msg="镜像%s创建成功" % (image_name,)))
def create(self, request, *args, **kwargs):
"""
创建镜像
:param request:
:param args:
:param kwargs:
:return:
"""
user = request.user
image_name = request.POST.get("image_name", "")
image_vul_name = request.POST.get("image_vul_name", "")
image_desc = request.POST.get("image_desc", "")
degree = request.POST.get("degree", "")
try:
image_rank = request.POST.get("rank", default=2.5)
image_rank = float(image_rank)
except:
image_rank = 2.5
try:
writeup_date = request.POST.get("writeup_date", "")
if writeup_date:
writeup_date = json.dumps(writeup_date)
except:
writeup_date = ""
is_flag = request.POST.get("is_flag", True)
image_file = request.FILES.get("file")
image_info = None
if image_name:
if ":" not in image_name:
image_name += ":latest"
image_info = ImageInfo.objects.filter(
image_name=image_name).first()
if not image_info:
image_info = ImageInfo(image_name=image_name,
image_vul_name=image_vul_name,
image_desc=image_desc,
rank=image_rank,
is_ok=False,
create_date=timezone.now(),
update_date=timezone.now(),
degree=degree,
writeup_date=writeup_date,
is_flag=is_flag)
if not image_file:
image_info.save()
task_id = tasks.create_image_task(image_info=image_info,
user_info=user,
request_ip=get_request_ip(request),
image_file=image_file)
if image_file:
task_info = TaskInfo.objects.filter(task_id=task_id).first()
task_msg = task_info.task_msg
return JsonResponse(json.loads(task_msg))
return JsonResponse(R.ok(task_id, msg="拉取镜像%s任务下发成功" % (image_name, )))
def create_image(task_id):
"""
创建镜像名称
"""
task_info = TaskInfo.objects.filter(task_id=task_id, task_status=1).first()
if not task_info:
return
operation_args = task_info.operation_args
args = json.loads(operation_args)
image_name = args["image_name"].strip()
image_desc = args["image_desc"].strip()
image_rank = args["rank"]
image_vul_name = args["image_vul_name"].strip()
image_info = ImageInfo.objects.filter(image_name=image_name).first()
if not image_info:
image_info = ImageInfo(image_name=image_name,
image_desc=image_desc,
rank=image_rank,
image_vul_name=image_vul_name)
image = None
msg = {}
try:
image = client.images.get(image_name)
except Exception as e:
image_info.is_ok = False
image_info.save()
try:
images = client.images.pull(image_name)
if Image == type(images):
image = images
else:
if len(images) > 0:
image = images[0]
except ImageNotFound:
msg = R.build(msg="%s 不存在")
except Exception:
traceback.print_exc()
msg = R.err(msg="%s 添加失败" % (image_name, ))
if image:
config = image.attrs["ContainerConfig"]
port_list = []
if "ExposedPorts" in config:
port_list = config["ExposedPorts"]
ports = []
for port in port_list:
port = port.replace("/", "").replace("tcp", "").replace("udp", "")
ports.append(port)
image_port = ",".join(ports)
image_info.image_port = image_port
image_info.is_ok = True
image_info.save()
msg = R.ok(msg="%s 添加成功" % (image_name, ),
data=json.dumps({"image_port": image_port}))
task_info.task_status = 3
else:
task_info.task_status = 4
task_info.task_msg = json.dumps(msg)
task_info.save()
def get_timing_imgs(request):
"""
获取官网镜像信息
"""
try:
url = "http://vulfocus.fofa.so/api/imgs/info"
res = requests.get(url, verify=False).content
req = json.loads(res)
image_names = list(ImageInfo.objects.all().values_list('image_name',
flat=True))
for item in req:
if item['image_name'] == "":
continue
if item['image_name'] in image_names:
if item['image_name'] == "vulfocus/vulfocus:latest":
continue
single_img = ImageInfo.objects.filter(
image_name__contains=item['image_name']).first()
if single_img.image_vul_name != item[
'image_vul_name'] or single_img.image_vul_name == "":
single_img.image_vul_name = item['image_vul_name']
if single_img.image_desc == "":
single_img.image_desc = item['image_desc']
if single_img.rank != item['rank']:
single_img.rank = item['rank']
if single_img.degree != item['degree']:
single_img.degree = json.dumps(item['degree'])
if "writeup_date" in item and single_img.writeup_date != item[
'writeup_date']:
single_img.writeup_date = item['writeup_date']
single_img.save()
else:
if "writeup_date" in item:
writeup_date = item['writeup_date']
else:
writeup_date = ""
image_info = ImageInfo(image_name=item['image_name'],
image_vul_name=item['image_vul_name'],
image_desc=item['image_desc'],
rank=item['rank'],
degree=json.dumps(item['degree']),
is_ok=False,
create_date=timezone.now(),
writeup_date=writeup_date,
update_date=timezone.now())
image_info.save()
return JsonResponse({"code": 200, "data": "成功"})
except Exception as e:
return JsonResponse({"code": 201, "data": e})
def update_images():
"""
定时获取dockerhub上的镜像信息
"""
url = "https://hub.docker.com/v2/repositories/vulfocus/?ordering=last_updated&page=1&page_size=1"
res = requests.get(url).content
req = json.loads(res)
count = req['count']
num = count / 100
for item_num in range(int(num) + 1):
new_url = "https://hub.docker.com/v2/repositories/vulfocus/?ordering=last_updated&page={}&page_size=100".format(
item_num + 1)
new_res = requests.get(new_url).content
new_req = json.loads(new_res)
if "results" not in new_req:
break
result = new_req['results']
image_names = list(ImageInfo.objects.all().values_list('image_name',
flat=True))
try:
for docker_image_info in result:
image_name = "vulfocus/%s" % (
docker_image_info['name'], ) + ':latest'
if image_name in image_names:
continue
if image_name == "vulfocus/vulfocus:latest":
continue
image_vul_name = image_name[:image_name.rfind(":")]
# 替换 CVE CNVD CNNVD _
if "cve" in image_vul_name:
image_vul_name = image_vul_name.replace("cve", "CVE")
if "cnvd" in image_vul_name:
image_vul_name = image_vul_name.replace("cnvd", "CNVD")
if "cnnvd" in image_vul_name:
image_vul_name = image_vul_name.replace("cnnvd", "CNNVD")
if "_" in image_vul_name:
image_vul_name = image_vul_name.replace("_", "-")
image_info = ImageInfo(image_name=image_name,
image_vul_name=image_vul_name,
image_desc="",
rank=2.5,
is_ok=False,
create_date=timezone.now(),
update_date=timezone.now())
image_info.save()
except Exception as e:
pass
def create_image(task_id):
"""
创建镜像名称
"""
task_info = TaskInfo.objects.filter(task_id=task_id, task_status=1).first()
if not task_info:
return
operation_args = task_info.operation_args
args = json.loads(operation_args)
image_name = args["image_name"].strip()
image_info = ImageInfo.objects.filter(image_name=image_name).first()
if not image_info:
image_desc = image_name
image_rank = 2.5
image_vul_name = image_desc
image_info = ImageInfo(image_name=image_name, image_desc=image_desc, rank=image_rank, image_vul_name=image_vul_name)
image = None
msg = {}
try:
image = client.images.get(image_name)
except Exception as e:
image_info.is_ok = False
image_info.save()
try:
last_info = {}
progress_info = {
"total": 0,
"progress_count": 0,
"progress": round(0.0, 2),
}
black_list = ["total", "progress_count", "progress"]
for line in api_docker_client.pull(image_name, stream=True, decode=True):
if "status" in line and "progressDetail" in line and "id" in line:
id = line["id"]
status = line["status"]
if len(line["progressDetail"]) > 0:
try:
current = line["progressDetail"]["current"]
total = line["progressDetail"]["total"]
line["progress"] = round((current / total) * 100, 2)
if (current / total) > 1:
line["progress"] = round(0.99 * 100, 2)
except:
line["progress"] = round(1 * 100, 2)
else:
if (("Download" in status or "Pull" in status) and ("complete" in status)) or ("Verifying" in status) or \
("Layer" in status and "already" in status and "exists" in status):
line["progress"] = round(100.00, 2)
else:
line["progress"] = round(0.00, 2)
progress_info[id] = line
progress_info["total"] = len(progress_info) - len(black_list)
progress_count = 0
for key in progress_info:
if key in black_list:
continue
if 100.00 != progress_info[key]["progress"]:
continue
progress_count += 1
progress_info["progress_count"] = progress_count
progress_info["progress"] = round((progress_count/progress_info["total"])*100, 2)
r.set(str(task_id), json.dumps(progress_info,ensure_ascii=False))
print(json.dumps(progress_info, ensure_ascii=False))
last_info = line
if "status" in last_info and ("Downloaded newer image for" in last_info["status"] or "Image is up to date for" in last_info["status"]):
image = client.images.get(image_name)
else:
raise Exception
except ImageNotFound:
msg = R.build(msg="%s 不存在" % (image_name,))
except Exception:
traceback.print_exc()
msg = R.err(msg="%s 添加失败" % (image_name,))
if image:
config = image.attrs["ContainerConfig"]
port_list = []
if "ExposedPorts" in config:
port_list = config["ExposedPorts"]
ports = []
for port in port_list:
port = port.replace("/", "").replace("tcp", "").replace("udp", "")
ports.append(port)
image_port = ",".join(ports)
image_info.image_port = image_port
image_info.is_ok = True
image_info.save()
msg = R.ok(msg="%s 添加成功" % (image_name,), data=json.dumps({"image_port": image_port}))
task_info.task_status = 3
else:
task_info.task_status = 4
task_info.task_msg = json.dumps(msg)
task_info.save()
def create_image_task(image_info, user_info, request_ip, image_file=None):
"""
创建镜像任务
"""
user_id = user_info.id
task_id = create_create_image_task(image_info=image_info, user_info=user_info)
if user_info.is_superuser:
image_name = image_info.image_name
image_desc = image_info.image_desc
image_vul_name = image_info.image_vul_name
image_rank = image_info.rank
task_info = TaskInfo.objects.filter(task_id=task_id).first()
if image_file:
task_msg = {}
try:
file_info = image_file.read()
images = client.images.load(file_info)
image = images[0]
repo_tags = image.attrs["RepoTags"]
if len(repo_tags) == 0:
# 移除本地镜像
try:
client.images.remove(image.id)
except Exception as e:
pass
task_msg = R.build(msg="文件镜像 Tag 不能为空")
else:
config = image.attrs["ContainerConfig"]
port_list = []
if "ExposedPorts" in config:
port_list = config["ExposedPorts"]
ports = []
for port in port_list:
port = port.replace("/", "").replace("tcp", "").replace("udp", "")
ports.append(port)
image_name = repo_tags[0]
image_port = ",".join(ports)
image_info = ImageInfo.objects.filter(image_name=image_name).first()
if not image_info:
image_info = ImageInfo()
image_info.image_name = image_name
image_info.image_port = image_port
# image_vul_name
image_info.image_vul_name = image_name.replace("vulfocus/","") if not image_vul_name else image_vul_name
# image_desc
image_info.image_desc = image_name.replace("vulfocus/","") if not image_desc else image_desc
# rank
image_info.rank = 2.5 if image_rank > 5 or image_rank < 0.5 else image_rank
image_info.is_ok = True
image_info.save()
task_info.task_name = "拉取镜像："+image_name
task_info.task_status = 3
task_msg = R.ok(data="%s 添加成功" % (image_name, ))
except Exception as e:
traceback.print_exc()
task_msg = R.err()
try:
image_info.delete()
except:
pass
task_info.task_status = 4
finally:
task_info.task_msg = json.dumps(task_msg)
task_info.update_date = timezone.now()
task_info.save()
elif image_name:
# 创建任务
# create_image(task_id=task_id)
create_image.delay(task_id)
else:
R.build(msg="镜像文件或镜像名称不能为空")
operation_args = ImageInfoSerializer(image_info).data
sys_log = SysLog(user_id=user_id, operation_type="镜像", operation_name="创建", ip=request_ip,
operation_value=operation_args["image_vul_name"], operation_args=json.dumps(operation_args))
sys_log.save()
else:
task_info = TaskInfo.objects.filter(task_id=task_id).first()
task_info.task_msg = json.dumps(R.build(msg="权限不足"))
task_info.task_status = 3
task_info.update_date = timezone.now()
task_info.save()
return task_id
def create(self, request, *args, **kwargs):
"""
创建镜像
:param request:
:param args:
:param kwargs:
:return:
"""
user = request.user
if not user.is_superuser:
return JsonResponse(R.build(msg="权限不足！"))
file = request.FILES.get("file")
rank = request.POST.get("rank", default=2.5)
image_name = request.POST.get("name")
try:
rank = float(rank)
except Exception as e:
return JsonResponse(R.build(msg="Rank 格式错误"))
image_vul_name = request.POST.get("vul_name", default="")
if not image_vul_name:
return JsonResponse(R.build(msg="漏洞名称不能为空"))
image_desc = request.POST.get("desc", default="")
if not file and not image_name:
return JsonResponse(R.build(msg="镜像文件或名称不能为空"))
if file and image_name:
return JsonResponse(R.build(msg="镜像文件或名称不能同时填写"))
image_port = ""
if not image_name and file:
try:
file_info = file.read()
images = client.images.load(file_info)
image = images[0]
repo_tags = image.attrs["RepoTags"]
if len(repo_tags) == 0:
# 移除本地镜像
try:
client.images.remove(image.id)
except Exception as e:
pass
return JsonResponse(msg="镜像名称不能为空")
config = image.attrs["ContainerConfig"]
port_list = []
if "ExposedPorts" in config:
port_list = config["ExposedPorts"]
ports = []
for port in port_list:
port = port.replace("/", "").replace("tcp", "").replace("udp", "")
ports.append(port)
image_name = repo_tags[0]
image_port = ",".join(ports)
except Exception as e:
return JsonResponse(R.err())
"""
查重
"""
count = ImageInfo.objects.filter(image_name=image_name).count()
if count > 0:
return JsonResponse(R.build(msg="漏洞镜像已存在!"))
if not file and image_name:
try:
image = client.images.get(image_name)
except Exception as e:
images = client.images.pull(image_name)
if Image == type(images):
image = images
else:
if len(images) > 0:
image = images[0]
else:
return JsonResponse(R.build("镜像不存在！"))
config = image.attrs["ContainerConfig"]
port_list = []
if "ExposedPorts" in config:
port_list = config["ExposedPorts"]
ports = []
for port in port_list:
port = port.replace("/", "").replace("tcp", "").replace("udp", "")
ports.append(port)
image_port = ",".join(ports)
image_info = ImageInfo(image_name=image_name, image_vul_name=image_vul_name, image_port=image_port, rank=rank,
image_desc=image_desc)
image_info.save()
rs_data = ImageInfoSerializer(image_info).data
request_ip = get_request_ip(request)
sys_log = SysLog(user_id=user.id, operation_type="镜像", operation_name="创建",
operation_value=rs_data["image_vul_name"], operation_args=rs_data, ip=request_ip)
sys_log.save()
return JsonResponse(R.ok(data=rs_data))
