def __init__(self):
"""Scanner verify initialization."""
self.scanner_name = "scanner-rpm-verify"
self.full_scanner_name = \
"registry.centos.org/pipeline-images/scanner-rpm-verify"
self.atomic_object = Atomic()
# Add logger
load_logger()
self.logger = logging.getLogger('scan-worker')
def __init__(self, *p, **k):
super(atomic_dbus, self).__init__(*p, **k)
self.atomic = Atomic()
self.tasks = []
self.tasks_lock = threading.Lock()
self.last_token = 0
self.scheduler_thread = threading.Thread(target = self.Scheduler)
self.scheduler_thread.daemon = True
self.scheduler_thread.start()
self.results = dict()
self.results_lock = threading.Lock()
def __init__(self, image, scanner, result_file):
# container/image under test
self.image = image
# scanner name / as installed /not full URL
self.scanner = scanner
# name of the output result file by scanner
self.result_file = result_file
# image_id
self.image_id = Atomic().get_input_id(self.image)
# set logger or set console
load_logger()
self.logger = logging.getLogger("scan-worker")
# Flag to indicate if image is mounted on local filesystem
self.is_mounted = False
# image mount path
self.image_mountpath = os.path.join("/", self.image_id)
# initialize the atomic mount object
self.mount_obj = mount.Mount()
# provide image id to mount object
self.mount_obj.image = self.image_id
# provide mount option read/write
self.mount_obj.options = ["rw"]
# provide mount point
self.mount_obj.mountpoint = self.image_mountpath
# set default for res_dir, this is a dir which is created
# by atomic, default result location
self.res_dir = None
def scan(self, image_under_test):
"""Run the pipleline scanner."""
self.image_under_test = image_under_test
self.image_id = Atomic().get_input_id(image_under_test)
self.image_rootfs_path = os.path.join("/", self.image_id)
json_data = {}
if not self.mount_image():
return False, json_data
cmd = [
'atomic',
'scan',
"--scanner={}".format(self.scanner_name),
"--rootfs={}".format(self.image_rootfs_path),
"{}".format(self.image_id)
]
self.logger.info("Executing atomic scan: {}".format(" ".join(cmd)))
process = subprocess.Popen(
cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE
)
out, err = process.communicate()
if out != "":
# TODO: hacky and ugly. figure a better way
# https://github.com/projectatomic/atomic/issues/577
output_json_file = os.path.join(
out.strip().split()[-1].split('.')[0],
"_{}".format(self.image_rootfs_path.split('/')[1]),
SCANNERS_OUTPUT[self.full_scanner_name][0]
)
self.logger.info("Result file: {}".format(output_json_file))
if os.path.exists(output_json_file):
json_data = json.loads(open(output_json_file).read())
else:
self.logger.critical(
"No scan results found at {}".format(output_json_file))
return False, json_data
else:
self.logger.warning(
"Failed to get the results from atomic CLI. Error:{}".format(
err)
)
return False, json_data
self.unmount_image(retries=10, delay=10)
shutil.rmtree(self.image_rootfs_path, ignore_errors=True)
self.logger.info(
"Finished executing scanner: {}".format(self.scanner_name))
return True, self.process_output(json_data)
def __init__(self, *p, **k):
super(atomic_dbus, self).__init__(*p, **k)
self.atomic = Atomic()
self.tasks = []
self.tasks_lock = threading.Lock()
self.last_token = 0
self.scheduler_thread = threading.Thread(target=self.Scheduler)
self.scheduler_thread.daemon = True
self.scheduler_thread.start()
self.results = dict()
self.results_lock = threading.Lock()
def __init__(self, image_under_test, scanner_name, full_scanner_name,
to_process_output):
# to be provided by child class
self.scanner_name = scanner_name
self.full_scanner_name = full_scanner_name
self.image_under_test = image_under_test
# Scanner class's own attributes
self.atomic_obj = Atomic()
self.image_id = self.atomic_obj.get_input_id(self.image_under_test)
self.to_process_output = to_process_output
def __init__(self, image_under_test, scanner_name, full_scanner_name,
to_process_output):
"""Scanner initialization."""
# to be provided by child class
self.scanner_name = scanner_name
self.full_scanner_name = full_scanner_name
self.image_under_test = image_under_test
# Scanner class's own attributes
self.image_id = Atomic().get_input_id(self.image_under_test)
self.to_process_output = to_process_output
# Add logger
load_logger()
self.logger = logging.getLogger('scan-worker')
def __init__(self):
self.scanner_name = "pipeline-scanner"
self.full_scanner_name = "registry.centos.org/pipeline-images/pipeline-scanner"
self.atomic_object = Atomic()
self.mount_object = mount.Mount()
class ScannerRPMVerify(object):
"""scanner-rpm-verify atomic scanner handler."""
def __init__(self):
"""Scanner verify initialization."""
self.scanner_name = "scanner-rpm-verify"
self.full_scanner_name = \
"registry.centos.org/pipeline-images/scanner-rpm-verify"
self.atomic_object = Atomic()
# Add logger
load_logger()
self.logger = logging.getLogger('scan-worker')
def run_atomic_scanner(self):
"""Run the atomic scan command."""
process = subprocess.Popen([
"atomic", "scan", "--scanner=rpm-verify", "{}".format(
self.image_id)
],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
# returns out, err
return process.communicate()
def scan(self, image_under_test):
"""
Run the scanner-rpm-verify atomic scanner.
Returns: Tuple stating the status of the execution and
actual data(True/False, json_data)
"""
self.image_under_test = image_under_test
self.image_id = self.atomic_object.get_input_id(self.image_under_test)
json_data = {}
out, err = self.run_atomic_scanner()
if out != "":
output_json_file = os.path.join(
out.strip().split()[-1].split('.')[0],
self.image_id,
# TODO: provision parsing multiple output files per scanner
SCANNERS_OUTPUT[self.full_scanner_name][0])
if os.path.exists(output_json_file):
json_data = json.loads(open(output_json_file).read())
else:
self.logger.critical(
"No scan results found at {}".format(output_json_file))
# FIXME: handle what happens in this case
return False, self.process_output(json_data)
else:
# TODO: do not exit here if one of the scanner failed to run,
# others might run
self.logger.critical(
"Error running the scanner {}. Error: {}".format(
self.scanner_name, err))
return False, self.process_output(json_data)
return True, self.process_output(json_data)
def process_output(self, json_data):
"""Process the output from scanner."""
data = {}
data["scanner_name"] = self.scanner_name
# TODO: More verifcation and validation on the data
data["msg"] = json_data["Summary"]
data["logs"] = json_data
return data
class atomic_dbus(slip.dbus.service.Object):
default_polkit_auth_required = "org.atomic.readwrite"
class Args:
def __init__(self):
self.image = None
self.recurse = False
self.debug = False
self.devices = None
self.driver = None
self.graph = None
self.force = None
self.import_location = None
self.export_location = None
self.compares = []
self.json = False
self.no_files = False
self.names_only = False
self.rpms = False
self.verbose = False
self.scan_targets = []
self.scanner = None
self.scan_type = None
self.list = False
self.rootfs = []
self.all = False
self.images = False
self.containers = False
self.container = False
self.prune = False
self.heading = False
self.truncate = False
def __init__(self, *p, **k):
super(atomic_dbus, self).__init__(*p, **k)
self.atomic = Atomic()
self.tasks = []
self.tasks_lock = threading.Lock()
self.last_token = 0
self.scheduler_thread = threading.Thread(target=self.Scheduler)
self.scheduler_thread.daemon = True
self.scheduler_thread.start()
self.results = dict()
self.results_lock = threading.Lock()
def Scheduler(self):
while True:
current_task = None
with self.tasks_lock:
if len(self.tasks) > 0:
current_task = self.tasks.pop(0)
if current_task is not None:
result = current_task[1].scan()
with self.results_lock:
self.results[current_task[0]] = result
time.sleep(1)
def AllocateToken(self):
with self.tasks_lock:
self.last_token += 1
return self.last_token
# The Version method takes in an image name and returns its version
# information
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="asb", out_signature="aa{sv}")
def Version(self, images, recurse=False):
versions = []
for image in images:
args = self.Args()
args.image = image
args.recurse = recurse
self.atomic.set_args(args)
versions.append({"Image": image, "Version": self.atomic.version()})
return versions
# The Verify method takes in an image name and returns whether or not the
# image should be updated
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="as", out_signature="s")
def Verify(self, images):
verifications = []
verify = Verify()
verify.useTTY = False
for image in images:
args = self.Args()
args.image = image
verify.set_args(args)
verifications.append({"Image": image, "Verification": verify.verify()}) # pylint: disable=no-member
return json.dumps(verifications)
# The StorageReset method deletes all containers and images from a system.
# Resets storage to its initial configuration.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature="", out_signature="")
def StorageReset(self):
storage = Storage()
# No arguments are passed for storage_reset function
args = self.Args()
storage.set_args(args)
storage.reset()
# The StorageImport method imports all containers and their associated
# contents from a filesystem directory.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature="ss", out_signature="")
def StorageImport(self, graph, import_location):
storage = Storage()
args = self.Args()
args.graph = graph
args.import_location = import_location
storage.set_args(args)
storage.Import()
# The StorageExport method exports all containers and their associated
# contents into a filesystem directory.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature="ssb", out_signature="")
def StorageExport(self, graph="/var/lib/docker", export_location="/var/lib/atomic/migrate", force=False):
storage = Storage()
args = self.Args()
args.graph = graph
args.export_location = export_location
args.force = force
storage.set_args(args)
storage.Export()
# The StorageModify method modifies the default storage setup.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature="asv", out_signature="")
def StorageModify(self, devices=None, driver=None):
storage = Storage()
args = self.Args()
if devices:
args.devices = devices
else:
args.devices = []
args.driver = driver
storage.set_args(args)
storage.modify()
# The Diff method shows differences between two container images, file
# diff or RPMS.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="ss", out_signature="s")
def Diff(self, first, second):
diff = Diff()
args = self.Args()
args.compares = [first, second]
args.json = True
args.no_files = False
args.names_only = False
args.rpms = True
args.verbose = True
diff.set_args(args)
return diff.diff()
# The ScanList method will return a list of all scanners.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="", out_signature="s")
def ScanList(self):
scan_list = Scan()
args = self.Args()
scan_list.set_args(args)
return scan_list.get_scanners_list()
# The ScanSetup method will create the scan object.
def _ScanSetup(self, scan_targets, scanner, scan_type, rootfs, _all, images, containers):
scan = Scan()
args = self.Args()
scan.useTTY = False
if scan_targets:
args.scan_targets = scan_targets
if scanner:
args.scanner = scanner
if scan_type:
args.scan_type = scan_type
if len(scan_targets):
args.scan_targets = scan_targets
args.rootfs = rootfs
args.all = _all
args.images = images
args.containers = containers
scan.set_args(args)
return scan
# The Scan method will return a string.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="asssasbbb", out_signature="s")
def Scan(self, scan_targets, scanner, scan_type, rootfs, _all, images, containers):
return self._ScanSetup(scan_targets, scanner, scan_type, rootfs, _all, images, containers).scan()
# The ScheduleScan method will return a token.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="asssasbbb", out_signature="x")
def ScheduleScan(self, scan_targets, scanner, scan_type, rootfs, _all, images, containers):
scan = self._ScanSetup(scan_targets, scanner, scan_type, rootfs, _all, images, containers)
token = self.AllocateToken()
with self.tasks_lock:
self.tasks.append((token, scan))
return token
# The GetScanResults method will determine whether or not the results for
# the token are ready.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="x", out_signature="s")
def GetScanResults(self, token):
with self.results_lock:
if token in self.results:
ret = self.results[token]
del self.results[token]
return ret
else:
return ""
# The Update method downloads the latest container image.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature="s", out_signature="")
def Update(self, image):
args = self.Args()
args.image = image
self.atomic.set_args(args)
self.atomic.update()
# The Images method will list all installed container images on the system.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="", out_signature="s")
def Images(self):
args = self.Args()
images = Images()
images.set_args(args)
return json.dumps(images.images())
# The Vulnerable method will send back information that says
# whether or not an installed container image is vulnerable
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="", out_signature="s")
def VulnerableInfo(self):
args = self.Args()
self.atomic.set_args(args)
return self.atomic.get_all_vulnerable_info()
# The containers.Ps method will list all containers on the system.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature="", out_signature="s")
def Ps(self):
ps = Containers()
ps.useTTY = False
args = self.Args()
ps.set_args(args)
return json.dumps(ps.ps())
def __init__(self, *p, **k):
super(atomic_dbus, self).__init__(*p, **k)
self.atomic = Atomic()
class atomic_dbus(slip.dbus.service.Object):
default_polkit_auth_required = "org.atomic.readwrite"
class Args():
def __init__(self):
self.image = None
self.recurse = False
self.debug = False
self.devices = None
self.driver = None
self.graph = None
self.force = None
self.import_location = None
self.export_location = None
self.compares = []
self.json = False
self.no_files = False
self.names_only = False
self.rpms = False
self.verbose = False
self.tty = True
def __init__(self, *p, **k):
super(atomic_dbus, self).__init__(*p, **k)
self.atomic = Atomic()
"""
The version method takes in an image name and returns its version
information
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='asb',
out_signature='aa{sv}')
def version(self, images, recurse=False):
versions = []
for image in images:
args = self.Args()
args.image = image
args.recurse = recurse
self.atomic.set_args(args)
versions.append({"Image": image,
"Version": self.atomic.version()})
return versions
"""
The verify method takes in an image name and returns whether or not the
image should be updated
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='as', out_signature='av')
def verify(self, images):
verifications = []
verify = Verify()
for image in images:
args = self.Args()
args.image = image
verify.set_args(args)
verifications.append({"Image": image,
"Verification": verify.verify()}) #pylint: disable=no-member
return verifications
"""
The storage_reset method deletes all containers and images from a system. Resets storage to its initial configuration.
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='', out_signature='')
def storage_reset(self):
storage = Storage()
# No arguments are passed for storage_reset function
args = self.Args()
storage.set_args(args)
storage.reset()
"""
The storage_import method imports all containers and their associated contents from a filesystem directory.
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='ss', out_signature='')
def storage_import(self, graph="/var/lib/docker", import_location="/var/lib/atomic/migrate"):
storage = Storage()
args = self.Args()
args.graph = graph
args.import_loc = import_location
storage.set_args(args)
storage.Import()
"""
The storage_export method exports all containers and their associated contents into a filesystem directory.
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='ssb', out_signature='')
def storage_export(self, graph="/var/lib/docker", export_location="/var/lib/atomic/migrate", force = False):
storage = Storage()
args = self.Args()
args.graph = graph
args.export_location = export_location
args.force = force
storage.set_args(args)
storage.Export()
"""
The storage_modify method modifies the default storage setup.
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='asv', out_signature='')
def storage_modify(self, devices=[], driver = None):
storage = Storage()
args = self.Args()
args.devices = devices
args.driver = driver
storage.set_args(args)
storage.modify()
"""
The diff method shows differences between two container images, file diff or RPMS.
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='ss',
out_signature='s')
def diff(self, first, second):
diff = Diff()
args = self.Args()
args.compares = [first, second]
args.json = True
args.no_files = False
args.names_only = False
args.rpms = True
args.verbose = True
diff.set_args(args)
return diff.diff()
"""
The get_scan_list method will return a list of all scanners.
"""
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='',
out_signature= 's')
def scan_list(self):
scan_list = Scan()
args = self.Args()
scan_list.set_args(args)
return scan_list.get_scanners_list()
def test_job_data(job_data):
msg = ""
logs = ""
json_data = None
logger.log(level=logging.INFO, msg="Received job data from tube")
logger.log(level=logging.INFO, msg="Job data: %s" % job_data)
# if job_data.get("tag") != None:
# image_full_name = job_data.get("name") + ":" + \
# job_data.get("image_tag")
# else:
# image_full_name = job_data.get("name")
# Receive and send `name_space` key-value as is
namespace = job_data.get('name_space')
image_full_name = job_data.get('name').split(":")[0] + ":" + \
job_data.get("tag")
logger.log(level=logging.INFO, msg="Pulling image %s" % image_full_name)
pull_data = conn.pull(
repository=image_full_name
)
if 'error' in pull_data:
logger.log(level=logging.FATAL, msg="Couldn't pull requested image")
logger.log(level=logging.FATAL, msg=pull_data)
raise
# logger.log(level=logging.INFO,
# msg="Creating container for image %s" % image_full_name)
# container = conn.create_container(image=image_full_name,
# command="yum -q check-update")
atomic_object = Atomic()
# get the SHA ID of the image.
image_id = atomic_object.get_input_id(image_full_name)
image_rootfs_path = os.path.join("/", image_id)
# create a directory /<image_id> where we'll mount image's rootfs
os.makedirs(image_rootfs_path)
# configure options before mounting the image rootfs
logger.log(level=logging.INFO,
msg="Setting up system to mount image's rootfs")
mount_object = mount.Mount()
mount_object.mountpoint = image_rootfs_path
mount_object.image = image_id
# mount the rootfs in read-write mode. else yum will fail
mount_object.options = ["rw"]
logger.log(level=logging.INFO,
msg="Mounting rootfs %s on %s" % (image_id, image_rootfs_path))
mount_object.mount()
logger.log(level=logging.INFO,
msg="Successfully mounted image's rootfs")
cmd = "atomic scan --scanner=%s --rootfs=%s %s" % \
("pipeline-scanner", image_rootfs_path, image_id)
logger.log(level=logging.INFO,
msg="Executing atomic scan: %s" % cmd)
process = subprocess.Popen([
'atomic',
'scan',
"--scanner=pipeline-scanner",
"--rootfs=%s" % image_rootfs_path,
"%s" % image_id
], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
out, err = process.communicate()
# logger.log(level=logging.INFO,
# msg="Created container with ID: %s" % container.get('Id'))
# conn.start(container=container.get('Id'))
# logger.log(level=logging.INFO,
# msg="Started container with ID: %s" % container.get('Id'))
# time.sleep(10)
# logs = conn.logs(container=container.get('Id'))
# logger.log(level=logging.INFO, msg="Stopping test container")
# conn.stop(container=container.get('Id'))
# logger.log(level=logging.INFO, msg="Removing the test container")
# conn.remove_container(container=container.get('Id'), force=True)
if out != "":
# TODO: hacky and ugly. figure a better way
output_json_file = os.path.join(
out.strip().split()[-1].split('.')[0],
"_%s" % image_rootfs_path.split('/')[1],
"image_scan_results.json"
)
if os.path.exists(output_json_file):
json_data = json.loads(open(output_json_file).read())
else:
logger.log(level=logging.FATAL,
msg="No scan results found at %s" % output_json_file)
raise
else:
logs = ""
logger.log(level=logging.INFO,
msg="Unmounting image's rootfs from %s" % image_rootfs_path)
mount_object.unmount()
os.rmdir(image_rootfs_path)
logger.log(level=logging.INFO, msg="Removing the image %s" % image_full_name)
conn.remove_image(image=image_full_name, force=True)
logger.log(level=logging.INFO, msg="Finished test...")
# if msg != "" and logs != "":
if json_data != None:
d = {
"image": image_full_name,
"msg": "Container image requires update",
"logs": json.dumps(json_data),
"action": "start_delivery",
"name_space": namespace
}
else:
d = {
"image": image_full_name,
"msg": "No updates required",
"logs": "",
"action": "start_delivery",
"name_space": namespace
}
bs.use("master_tube")
jid = bs.put(json.dumps(d))
logger.log(
level=logging.INFO,
msg="Put job on master tube with id: %d" % jid
)
class atomic_dbus(slip.dbus.service.Object):
default_polkit_auth_required = "org.atomic.readwrite"
class Args():
def __init__(self):
self.image = None
self.recurse = False
self.debug = False
self.devices = None
self.driver = None
self.graph = None
self.force = None
self.import_location = None
self.export_location = None
self.compares = []
self.json = False
self.no_files = False
self.names_only = False
self.rpms = False
self.verbose = False
self.scan_targets = []
self.scanner = None
self.scan_type = None
self.list = False
self.rootfs = []
self.all = False
self.images = False
self.containers = False
self.container = False
self.prune = False
self.heading = False
self.truncate = False
def __init__(self, *p, **k):
super(atomic_dbus, self).__init__(*p, **k)
self.atomic = Atomic()
self.tasks = []
self.tasks_lock = threading.Lock()
self.last_token = 0
self.scheduler_thread = threading.Thread(target = self.Scheduler)
self.scheduler_thread.daemon = True
self.scheduler_thread.start()
self.results = dict()
self.results_lock = threading.Lock()
def Scheduler(self):
while True:
current_task = None
with self.tasks_lock:
if(len(self.tasks) > 0):
current_task = self.tasks.pop(0)
if current_task is not None:
result = current_task[1].scan()
with self.results_lock:
self.results[current_task[0]] = result
sleep(1)
def AllocateToken(self):
with self.tasks_lock:
self.last_token += 1
return self.last_token
# The Version method takes in an image name and returns its version
# information
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='asb',
out_signature='aa{sv}')
def Version(self, images, recurse=False):
versions = []
for image in images:
args = self.Args()
args.image = image
args.recurse = recurse
self.atomic.set_args(args)
versions.append({"Image": image,
"Version": self.atomic.version()})
return versions
# The Verify method takes in an image name and returns whether or not the
# image should be updated
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='as', out_signature='s')
def Verify(self, images):
verifications = []
verify = Verify()
verify.useTTY = False
for image in images:
args = self.Args()
args.image = image
verify.set_args(args)
verifications.append({"Image": image,
"Verification": verify.verify()}) #pylint: disable=no-member
return json.dumps(verifications)
# The StorageReset method deletes all containers and images from a system.
# Resets storage to its initial configuration.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature='', out_signature='')
def StorageReset(self):
storage = Storage()
# No arguments are passed for storage_reset function
args = self.Args()
storage.set_args(args)
storage.reset()
# The StorageImport method imports all containers and their associated
# contents from a filesystem directory.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature='ss', out_signature='')
def StorageImport(self, graph, import_location):
storage = Storage()
args = self.Args()
args.graph = graph
args.import_location = import_location
storage.set_args(args)
storage.Import()
# The StorageExport method exports all containers and their associated
# contents into a filesystem directory.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature='ssb', out_signature='')
def StorageExport(self, graph="/var/lib/docker", export_location="/var/lib/atomic/migrate", force = False):
storage = Storage()
args = self.Args()
args.graph = graph
args.export_location = export_location
args.force = force
storage.set_args(args)
storage.Export()
# The StorageModify method modifies the default storage setup.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature='asv', out_signature='')
def StorageModify(self, devices=None, driver = None):
storage = Storage()
args = self.Args()
if devices:
args.devices = devices
else:
args.devices = []
args.driver = driver
storage.set_args(args)
storage.modify()
# The Diff method shows differences between two container images, file
# diff or RPMS.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='ss',
out_signature='s')
def Diff(self, first, second):
diff = Diff()
args = self.Args()
args.compares = [first, second]
args.json = True
args.no_files = False
args.names_only = False
args.rpms = True
args.verbose = True
diff.set_args(args)
return diff.diff()
# The ScanList method will return a list of all scanners.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='',
out_signature= 's')
def ScanList(self):
scan_list = Scan()
args = self.Args()
scan_list.set_args(args)
return scan_list.get_scanners_list()
# The LastScanned method will return the time of the last scan which was done.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='', out_signature='s')
def LastScanned(self):
scan = Scan()
args = self.Args()
scan.set_args(args)
last_scanned = scan.get_last_time_all_scanned()
if last_scanned:
return last_scanned.strftime("%Y-%m-%d %H:%M:%S")
return "Last scan time was not found"
# The ScanSetup method will create the scan object.
def _ScanSetup(self, scan_targets, scanner, scan_type, rootfs, _all, images, containers):
scan = Scan()
args = self.Args()
scan.useTTY = False
if scan_targets:
args.scan_targets = scan_targets
if scanner:
args.scanner = scanner
if scan_type:
args.scan_type = scan_type
if len(scan_targets):
args.scan_targets = scan_targets
args.rootfs = rootfs
args.all = _all
args.images = images
args.containers = containers
scan.set_args(args)
return scan
# The Scan method will return a string.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='asssasbbb',
out_signature= 's')
def Scan(self, scan_targets, scanner, scan_type, rootfs, _all, images, containers):
return self._ScanSetup(scan_targets, scanner, scan_type, rootfs, _all, images, containers).scan()
# The ScheduleScan method will return a token.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='asssasbbb',
out_signature= 'x')
def ScheduleScan(self, scan_targets, scanner, scan_type, rootfs, _all, images, containers):
scan = self._ScanSetup(scan_targets, scanner, scan_type, rootfs, _all, images, containers)
token = self.AllocateToken()
with self.tasks_lock:
self.tasks.append((token, scan))
return token
# The GetScanResults method will determine whether or not the results for
# the token are ready.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='x',
out_signature= 's')
def GetScanResults(self, token):
with self.results_lock:
if token in self.results:
ret = self.results[token]
del self.results[token]
return ret
else:
return ""
# The Update method downloads the latest container image.
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature='s', out_signature='')
def Update(self, image):
args = self.Args()
args.image = image
self.atomic.set_args(args)
self.atomic.update()
# The Images method will list all installed container images on the system.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='', out_signature='s')
def Images(self):
args = self.Args()
self.atomic.set_args(args)
return json.dumps(self.atomic.images())
# The Vulnerable method will send back information that says
# whether or not an installed container image is vulnerable
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='', out_signature='s')
def VulnerableInfo(self):
args = self.Args()
self.atomic.set_args(args)
return self.atomic.get_all_vulnerable_info()
# The Ps method will list all containers on the system.
@slip.dbus.polkit.require_auth("org.atomic.read")
@dbus.service.method("org.atomic", in_signature='', out_signature='s')
def Ps(self):
ps = Ps()
args = self.Args()
ps.set_args(args)
return json.dumps(ps.ps())
class PipelineScanner(object):
"""
pipeline-scanner atomic scanner handler
"""
def __init__(self):
self.scanner_name = "pipeline-scanner"
self.full_scanner_name = "registry.centos.org/pipeline-images/pipeline-scanner"
self.atomic_object = Atomic()
self.mount_object = mount.Mount()
def mount_image(self):
"""
Mount image on local file system
"""
# get the SHA ID of the image.
self.mount_object.mountpoint = self.image_rootfs_path
self.mount_object.image = self.image_id
# mount the rootfs in read-write mode. else yum will fail
self.mount_object.options = ["rw"]
# configure options before mounting the image rootfs
logger.log(level=logging.INFO,
msg="Setting up system to mount image's rootfs")
# create a directory /<image_id> where we'll mount image's rootfs
try:
os.makedirs(self.image_rootfs_path)
except OSError as error:
logger.log(
level=logging.WARNING,
msg=str(error)
)
logger.log(
level=logging.INFO,
msg="Unmounting and removing directory %s " % self.image_rootfs_path
)
try:
self.mount_object.unmount()
except Exception as e:
logger.log(
level=logging.WARNING,
msg="Failed to unmount path= %s - Error: %s" % (self.image_rootfs_path, str(e))
)
else:
try:
shutil.rmtree(self.image_rootfs_path)
except Exception as e:
logger.log(
level=logging.WARNING,
msg="Failed to remove= %s - Error: %s" % (self.image_rootfs_path, str(e))
)
# retry once more
try:
os.makedirs(self.image_rootfs_path)
except OSError as error:
logger.log(
level=logging.FATAL,
msg=str(error)
)
# fail! and return
return False
logger.log(
level=logging.INFO,
msg="Mounting rootfs %s on %s" % (self.image_id, self.image_rootfs_path))
self.mount_object.mount()
logger.log(level=logging.INFO,
msg="Successfully mounted image's rootfs")
return True
def unmount_image(self):
"""
Unmount image using the Atomic mount object
"""
logger.log(
level=logging.INFO,
msg="Unmounting image's rootfs from %s" % self.mount_object.mountpoint)
# TODO: Error handling and logging
self.mount_object.unmount()
def remove_image(self):
"""
Removes the mounted image rootfs path as well as
removes the image using docker
"""
try:
shutil.rmtree(self.image_rootfs_path)
except OSError as error:
logger.log(
level=logging.WARNING,
msg="Error removing image rootfs path. %s" % str(error)
)
logger.log(
level=logging.INFO,
msg="Removing the image %s" % self.image_under_test)
conn.remove_image(image=self.image_under_test, force=True)
def run(self, image_under_test):
"""
Run the scanner
"""
self.image_under_test = image_under_test
self.image_id = self.atomic_object.get_input_id(image_under_test)
self.image_rootfs_path = os.path.join("/", self.image_id)
json_data = {}
if not self.mount_image():
return False, json_data
cmd = [
'atomic',
'scan',
"--scanner=%s" % self.scanner_name,
"--rootfs=%s" % self.image_rootfs_path,
"%s" % self.image_id
]
logger.log(
level=logging.INFO,
msg="Executing atomic scan: %s" % " ".join(cmd)
)
process = subprocess.Popen(
cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE
)
out, err = process.communicate()
if out != "":
# TODO: hacky and ugly. figure a better way
# https://github.com/projectatomic/atomic/issues/577
output_json_file = os.path.join(
out.strip().split()[-1].split('.')[0],
"_%s" % self.image_rootfs_path.split('/')[1],
SCANNERS_OUTPUT[self.full_scanner_name][0]
)
logger.log(
level=logging.INFO,
msg="Result file: %s" % output_json_file
)
if os.path.exists(output_json_file):
json_data = json.loads(open(output_json_file).read())
else:
logger.log(level=logging.FATAL,
msg="No scan results found at %s" % output_json_file)
return False, json_data
else:
logger.log(
level=logging.WARNING,
msg="Failed to get the results from atomic CLI. Error:%s" % err
)
return False, json_data
self.unmount_image()
shutil.rmtree(self.image_rootfs_path, ignore_errors=True)
logger.log(
level=logging.INFO,
msg="Finished executing scanner: %s" % self.scanner_name)
return True, self.process_output(json_data)
def process_output(self, json_data):
"""
Process the output from the scanner, parse the json and
add meaningful information based on validations
"""
data = {}
data["scanner_name"] = self.scanner_name
if json_data["Scan Results"]["Package Updates"]:
data["logs"] = json_data
data["msg"] = "Container image requires update."
else:
data["logs"] = {}
data["msg"] = "No updates required."
return data
def __init__(self):
self.scanner_name = "scanner-rpm-verify"
self.full_scanner_name = "registry.centos.org/pipeline-images/scanner-rpm-verify"
self.atomic_object = Atomic()
class ScannerRPMVerify(object):
"""
scanner-rpm-verify atomic scanner handler
"""
def __init__(self):
self.scanner_name = "scanner-rpm-verify"
self.full_scanner_name = "registry.centos.org/pipeline-images/scanner-rpm-verify"
self.atomic_object = Atomic()
def run_atomic_scanner(self):
"""
Run the atomic scan command
"""
process = subprocess.Popen([
"atomic",
"scan",
"--scanner=rpm-verify",
"%s" % self.image_id
], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
# returns out, err
return process.communicate()
def run(self, image_under_test):
"""
Run the scanner-rpm-verify atomic scanner
Returns: Tuple stating the status of the execution and
actual data
(True/False, json_data)
"""
self.image_under_test = image_under_test
self.image_id = self.atomic_object.get_input_id(self.image_under_test)
json_data = {}
out, err = self.run_atomic_scanner()
if out != "":
output_json_file = os.path.join(
out.strip().split()[-1].split('.')[0],
self.image_id,
# TODO: provision parsing multiple output files per scanner
SCANNERS_OUTPUT[self.full_scanner_name][0]
)
if os.path.exists(output_json_file):
json_data = json.loads(open(output_json_file).read())
else:
logger.log(
level=logging.FATAL,
msg="No scan results found at %s" % output_json_file)
# FIXME: handle what happens in this case
return False, json_data
else:
# TODO: do not exit here if one of the scanner failed to run,
# others might run
logger.log(
level=logging.FATAL,
msg="Error running the scanner %s. Error: %s" % (self.scanner_name, err)
)
return False, json_data
return True, self.process_output(json_data)
def process_output(self, json_data):
"""
Process the output from scanner
"""
data = {}
data["scanner_name"] = self.scanner_name
# TODO: More verifcation and validation on the data
data["msg"] = "RPM verify results."
data["logs"] = json_data
return data
class Scanner(object):
def __init__(self, image_under_test, scanner_name, full_scanner_name,
to_process_output):
# to be provided by child class
self.scanner_name = scanner_name
self.full_scanner_name = full_scanner_name
self.image_under_test = image_under_test
# Scanner class's own attributes
self.atomic_obj = Atomic()
self.image_id = self.atomic_obj.get_input_id(self.image_under_test)
self.to_process_output = to_process_output
def run_atomic_scanner(self, cmd):
"""
Run the scanner with the cmd provided by child class
"""
process = subprocess.Popen(
cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE
)
# returns out, err
return process.communicate()
def run(self, cmd):
# self.image_under_test = image_under_test
# self.image_id = self.atomic_obj.get_input_id(self.image_under_test)
json_data = {}
out, err = self.run_atomic_scanner(cmd)
if out != "":
output_json_file = os.path.join(
out.strip().split()[-1].split('.')[0],
self.image_id,
SCANNERS_OUTPUT[self.full_scanner_name][0]
)
if os.path.exists(output_json_file):
json_data = json.loads(open(output_json_file).read())
else:
logger.log(
level=logging.FATAL,
msg="No scan results found at {}".format(output_json_file)
)
return False, json_data
else:
logger.log(
level=logging.FATAL,
msg="Error running the scanner {}. Error {}".format(
self.scanner_name,
err
)
)
return False, json_data
if self.to_process_output:
return True, self.process_output(json_data)
return True, json_data
def process_output(self, json_data):
"""
Process the output from scanner
"""
data = {}
data["scanner_name"] = self.scanner_name
data["msg"] = "{} results.".format(self.scanner_name)
data["logs"] = json_data
return data
