def build(self, tags: List[str]):
print("Building service: {}".format(self.name, ))
tags = tags or ['latest']
build_tag = "{registry}/{image}:{tag}".format(registry=self._registry,
image=self._image,
tag="_build_")
try:
docker.build(
self._context_dir,
file=self._docker_file,
tag=build_tag,
_out=_p,
_err=_e,
_out_bufsize=0,
)
for tag in tags:
reg_tag = "{registry}/{image}:{tag}".format(
registry=self._registry, image=self._image, tag=tag)
print("Tagged: {}".format(reg_tag))
docker.tag(build_tag,
reg_tag,
_out=_p,
_err=_e,
_out_bufsize=0)
docker.rmi(build_tag, _out=_p)
except sh.ErrorReturnCode_1:
pass
def remove_images(ctx, images, force=False):
for image in images:
ctx.log('docker removing image %s' % image)
if force:
print(docker.rmi('-f', image))
else:
print(docker.rmi(image))
def remove_images(ctx, images, force=False):
for image in images:
ctx.log('docker removing image %s' % image)
if force:
print(docker.rmi('-f', image))
else:
print(docker.rmi(image))
def cli(ctx, images, private_registry):
"""pull images from private registry
images could be a list param
"""
ctx.log('pull %s from %s' % (images, private_registry))
for image in images:
pulled_image = parse_image(image, private_registry)
ctx.log('RUN: docker pull %s' % pulled_image)
try:
docker.pull(pulled_image)
if private_registry:
docker.tag(pulled_image, image)
docker.rmi(pulled_image)
except:
ctx.log('failed pull %s' % image)
print(docker.images())
def cli(ctx, images, all, private_registry):
"""push images to private registry
images could be a list param
"""
if all:
images = docker.images('--format','{{.Repository}}:{{.Tag}}')
images = images.split()
print(images)
ctx.log('push %s to %s' % (images, private_registry or 'DockerHub'))
for image in images:
ctx.log('RUN: docker push %s' % image)
if private_registry:
new_image = '%s/%s' % (private_registry, image)
docker.tag(image, new_image)
print(docker.push(new_image))
docker.rmi(new_image)
else:
print(docker.push(image))
def cli(ctx, images, all, private_registry):
"""push images to private registry
images could be a list param
"""
if all:
images = docker.images('--format', '{{.Repository}}:{{.Tag}}')
images = images.split()
print(images)
ctx.log('push %s to %s' % (images, private_registry or 'DockerHub'))
for image in images:
ctx.log('RUN: docker push %s' % image)
if private_registry:
new_image = '%s/%s' % (private_registry, image)
docker.tag(image, new_image)
print(docker.push(new_image))
docker.rmi(new_image)
else:
print(docker.push(image))
def remove_image(tag: str):
with sudo:
docker.rmi(tag, _fg=True)
def delete_image(self):
delete_command = docker.rmi(self.image_name)
if delete_command.exit_code == 0:
self.image_built = False
def reset(package: str):
container_name = package + "_build"
image_name = package + "_image"
docker.rm("-f", container_name, _ok_code=[0, 1])
docker.rmi(image_name, _ok_code=[0, 1])
def run_inference(self,
docker_name,
package: str,
docker_args: [str],
fuzzer_image: str,
build_file: str,
inference_command_args: List[str],
timeout_per_package: float,
qemu: bool = False):
"""
:param self:
:param docker_name:
:param package:
:param docker_args:
:param fuzzer_image:
:param build_file:
:param inference_command_args:
:param timeout_per_package:
:type inference_command_args: List
:return:
"""
inference_command = None
from celery.platforms import signals
def int_handler(signum, frame):
print("Int handler!")
if inference_command is not None:
try:
docker_command.stop(
docker_name, _timeout=120
) # It should not take longer than 120 seconds to kill a docker container, right????
except sh.ErrorReturnCode:
return True
except sh.TimeoutException: # It took too long too kill the docker container - we are going to ignore that for now, we want to continue fuzzing
return True
return True
# fuzzer_command.kill()
# fuzzer_command.wait()
else:
return True
signals['INT'] = int_handler
print("Now working on {0}".format(package))
try:
if os.path.exists(build_file):
with open(build_file, "r") as jsonfp:
build_dict = json.load(jsonfp)
package_image_name = build_dict["docker_image_name"]
else:
package_image_name = package + "_" + str(uuid.uuid4())[:8]
if not os.path.exists(os.path.dirname(build_file)):
os.mkdir(os.path.dirname(build_file))
# TODO: There is an issue with qemu here. Fix this!
package_image_name = helpers.docker_builder.return_current_package_image(
package=package,
fuzzer_image=fuzzer_image,
package_image=package_image_name,
json_output_path=build_file,
qemu=qemu)
print("docker run", " ".join(docker_args), package_image_name,
" ".join(map(lambda x: str(x), inference_command_args)))
build_dict = {}
with open(build_file) as build_filefp:
build_dict = json.load(build_filefp)
if build_dict["qemu"] and "-Q" not in inference_command_args:
inference_command_args.append("-Q")
elif not build_dict["qemu"] and "-Q" in inference_command_args:
inference_command_args.remove("-Q")
docker_args.insert(0, '--cpus=1.0')
inference_command = docker_command.run(
docker_args,
package_image_name,
inference_command_args,
_out=sys.stdout,
_timeout=timeout_per_package) # type: sh.RunningCommand
if inference_command.exit_code != 0:
print("Some went wrong for package {0}", package)
return False
if not KEEP_IMAGES:
docker_command.rmi("-f", package_image_name)
print("Done! Returning True")
return True
except sh.ErrorReturnCode as e:
print("Inference error:")
print("STDOUT:\n", e.stdout.decode("utf-8"))
print("STDERR:\n", e.stderr.decode("utf-8"))
print("command line: {0}".format(e.full_cmd))
logger.error("Inference error:")
logger.error("STDOUT:\n", e.stdout.decode("utf-8"))
logger.error("STDERR:\n", e.stderr.decode("utf-8"))
logger.error("command line: {0}".format(e.full_cmd))
return False
except sh.TimeoutException as e:
print("Inferring {0} timed out... Next one!".format(package))
return True
except sh.SignalException_SIGKILL as e:
print("Killed")
return True
return True